diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat new file mode 100644 index 00000000000..379eab90063 --- /dev/null +++ b/.ci/scripts/install-tools.bat @@ -0,0 +1,16 @@ +set GOPATH=%WORKSPACE% +set MAGEFILE_CACHE=%WORKSPACE%\.magefile +set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH% + +where /q curl +IF ERRORLEVEL 1 ( + choco install curl -y --no-progress --skipdownloadcache +) +mkdir %WORKSPACE%\bin +where /q gvm +IF ERRORLEVEL 1 ( + curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe +) +FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i + +go install github.com/elastic/beats/vendor/github.com/magefile/mage diff --git a/.travis.yml b/.travis.yml index 05f3f701f3d..345fd20c1ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -235,7 +235,7 @@ before_install: - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - - if [ $TRAVIS_OS_NAME = osx ]; then pip install virtualenv; fi + - if [ $TRAVIS_OS_NAME = osx ]; then pip install virtualenv==16.7.9; fi # Skips installations step diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 233432fda3e..3bae809bc2e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -24,10 +24,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Journalbeat* +- Improve parsing of syslog.pid in journalbeat to strip the username when present {pull}16116[16116] + *Metricbeat* - Make use of secure port when accessing Kubelet API {pull}16063[16063] +- Add Tomcat overview dashboard {pull}14026[14026] *Packetbeat* @@ -51,7 +54,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update replicaset group to apps/v1 {pull}15854[15802] - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} -- Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] +- Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] +- Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] *Auditbeat* @@ -66,6 +70,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942] - Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900] - Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088] +- Fix a connection error in httpjson input. {pull}16123[16123] *Heartbeat* @@ -82,6 +87,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change lookup_fields from metricset.host to service.address {pull}15883[15883] - Add dedot for cloudwatch metric name. {issue}15916[15916] {pull}15917[15917] - Fixed issue `logstash-xpack` module suddenly ceasing to monitor Logstash. {issue}15974[15974] {pull}16044[16044] +- Fix skipping protocol scheme by light modules. {pull}16205[pull] *Packetbeat* @@ -98,17 +104,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - Add document_id setting to decode_json_fields processor. {pull}15859[15859] - +- Add `aws_ec2` provider for autodiscover. {issue}12518[12518] {pull}14823[14823] *Auditbeat* *Filebeat* + - Set event.outcome field based on googlecloud audit log output. {pull}15731[15731] - Add dashboard for AWS ELB fileset. {pull}15804[15804] - Add dashboard for AWS vpcflow fileset. {pull}16007[16007] - Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936] - Add custom string mapping to CEF module to support Forcepoint NGFW {issue}14663[14663] {pull}15910[15910] +- Add ingress nginx controller fileset {pull}16197[16197] +- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836] +- Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121] *Heartbeat* @@ -141,6 +151,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add collecting AuroraDB metrics in rds metricset. {issue}14142[14142] {pull}16004[16004] - Reuse connections in SQL module. {pull}16001[16001] - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795] +- Add kubernetes storage class support via kube-state-metrics. {pull}16145[16145] *Packetbeat* diff --git a/Jenkinsfile b/Jenkinsfile index ee1b4434b5a..6033665066e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,12 +17,14 @@ pipeline { disableResume() durabilityHint('PERFORMANCE_OPTIMIZED') disableConcurrentBuilds() +// checkoutToSubdirectory "${env.BASE_DIR}" } triggers { issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') } parameters { booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') } stages { /** @@ -31,38 +33,24 @@ pipeline { stage('Checkout') { options { skipDefaultCheckout() } steps { + deleteDir() gitCheckout(basedir: "${BASE_DIR}") - stash allowEmpty: true, name: 'source', useDefaultExcludes: false - script { - env.GO_VERSION = readFile("${BASE_DIR}/.go-version").trim() - env.BUILD_FILEBEAT = isChanged(["^filebeat/.*"]) - env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"]) - env.BUILD_AUDITBEAT = isChanged(["^auditbeat/.*"]) - env.BUILD_METRICBEAT = isChanged(["^metricbeat/.*"]) - env.BUILD_PACKETBEAT = isChanged(["^packetbeat/.*"]) - env.BUILD_WINLOGBEAT = isChanged(["^winlogbeat/.*"]) - env.BUILD_DOCKERLOGBEAT = isChanged(["^x-pack/dockerlogbeat/.*"]) - env.BUILD_FUNCTIONBEAT = isChanged(["^x-pack/functionbeat/.*"]) - env.BUILD_JOURNALBEAT = isChanged(["^journalbeat/.*"]) - env.BUILD_GENERATOR = isChanged(["^generator/.*"]) - env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/*"]) - env.BUILD_DOCS = isGitRegionMatch(patterns: ["^docs/.*"], comparator: 'regexp') || params.runAllStages - env.BUILD_LIBBEAT = isGitRegionMatch(patterns: ["^Libbeat/.*"], comparator: 'regexp') || params.runAllStages + dir("${BASE_DIR}"){ + loadConfigEnvVars() } + stash allowEmpty: true, name: 'source', useDefaultExcludes: false } } stage('Lint'){ options { skipDefaultCheckout() } steps { - withBeatsEnv(){ - makeTarget("Lint", "check") - } + makeTarget("Lint", "check") } } stage('Build and Test'){ failFast false parallel { - stage('Filebeat'){ + stage('Filebeat oss'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { @@ -71,31 +59,49 @@ pipeline { return env.BUILD_FILEBEAT != "false" } } - stages { - stage('Filebeat oss'){ - steps { - withBeatsEnv(){ - makeTarget("Filebeat oss Linux", "-C filebeat testsuite") - } - } + steps { + makeTarget("Filebeat oss Linux", "-C filebeat testsuite") + } + } + stage('Filebeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT_XPACK != "false" } - stage('Filebeat x-pack'){ - steps { - withBeatsEnv(){ - makeTarget("Filebeat x-pack Linux", "-C x-pack/filebeat testsuite") - } - } + } + steps { + makeTarget("Filebeat x-pack Linux", "-C x-pack/filebeat testsuite") + } + } + stage('Filebeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" } - stage('Filebeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - steps { - withBeatsEnv(){ - makeTarget("Filebeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C filebeat testsuite") - } - } + } + steps { + makeTarget("Filebeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C filebeat testsuite") + } + } + stage('Filebeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && params.windowsTest } } + steps { + mageTargetWin("Filebeat oss Windows Unit test", "-d filebeat goUnitTest") + //mageTargetWin("Filebeat oss Windows Integration test", "-d filebeat goIntegTest") + } } stage('Heartbeat'){ agent { label 'ubuntu && immutable' } @@ -109,23 +115,32 @@ pipeline { stages { stage('Heartbeat oss'){ steps { - withBeatsEnv(){ - makeTarget("Heartbeat oss Linux", "-C heartbeat testsuite") - } + makeTarget("Heartbeat oss Linux", "-C heartbeat testsuite") } } stage('Heartbeat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } steps { - withBeatsEnv(){ - makeTarget("Heartbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C heartbeat testsuite") + makeTarget("Heartbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C heartbeat testsuite") + } + } + stage('Heartbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest } } + steps { + mageTargetWin("Heartbeat oss Windows Unit test", "-d heartbeat goTestUnit") + } } } } - stage('Auditbeat'){ + stage('Auditbeat oss'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { @@ -135,38 +150,52 @@ pipeline { } } stages { - stage('Auditbeat oss'){ + stage('Auditbeat Linux'){ steps { - withBeatsEnv(){ - makeTarget("Auditbeat oss Linux", "-C auditbeat testsuite") - } + makeTarget("Auditbeat oss Linux", "-C auditbeat testsuite") } } stage('Auditbeat crosscompile'){ steps { - withBeatsEnv(){ - makeTarget("Auditbeat oss crosscompile", "-C auditbeat crosscompile") - } - } - } - stage('Auditbeat x-pack'){ - steps { - withBeatsEnv(){ - makeTarget("Auditbeat x-pack Linux", "-C x-pack/auditbeat testsuite") - } + makeTarget("Auditbeat oss crosscompile", "-C auditbeat crosscompile") } } stage('Auditbeat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } steps { - withBeatsEnv(){ - makeTarget("Auditbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C auditbeat testsuite") + makeTarget("Auditbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C auditbeat testsuite") + } + } + stage('Auditbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest } } + steps { + mageTargetWin("Auditbeat Windows Unit test", "-d auditbeat goUnitTest") + //mageTargetWin("Auditbeat Windows Integration test", "-d auditbeat goIntegTest") + } } } } + stage('Auditbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT_XPACK != "false" + } + } + steps { + makeTarget("Auditbeat x-pack Linux", "-C x-pack/auditbeat testsuite") + } + } stage('Libbeat'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } @@ -179,33 +208,33 @@ pipeline { stages { stage('Libbeat oss'){ steps { - withBeatsEnv(){ - makeTarget("Libbeat oss Linux", "-C libbeat testsuite") - } + makeTarget("Libbeat oss Linux", "-C libbeat testsuite") } } stage('Libbeat crosscompile'){ steps { - withBeatsEnv(){ - makeTarget("Libbeat oss crosscompile", "-C libbeat crosscompile") - } + makeTarget("Libbeat oss crosscompile", "-C libbeat crosscompile") } } stage('Libbeat stress-tests'){ steps { - withBeatsEnv(){ - makeTarget("Libbeat stress-tests", "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") - } + makeTarget("Libbeat stress-tests", "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") } } - stage('Libbeat x-pack'){ - steps { - withBeatsEnv(){ - makeTarget("Libbeat x-pack Linux", "-C x-pack/libbeat testsuite") - } - } + } + } + stage('Libbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT_XPACK != "false" } } + steps { + makeTarget("Libbeat x-pack Linux", "-C x-pack/libbeat testsuite") + } } stage('Metricbeat Unit tests'){ agent { label 'ubuntu && immutable' } @@ -217,9 +246,7 @@ pipeline { } } steps { - withBeatsEnv(){ - makeTarget("Metricbeat Unit tests", "-C metricbeat unit-tests coverage-report") - } + makeTarget("Metricbeat Unit tests", "-C metricbeat unit-tests coverage-report") } } stage('Metricbeat Integration tests'){ @@ -232,9 +259,7 @@ pipeline { } } steps { - withBeatsEnv(){ - makeTarget("Metricbeat Integration tests", "-C metricbeat integration-tests-environment coverage-report") - } + makeTarget("Metricbeat Integration tests", "-C metricbeat integration-tests-environment coverage-report") } } stage('Metricbeat System tests'){ @@ -247,24 +272,20 @@ pipeline { } } steps { - withBeatsEnv(){ - makeTarget("Metricbeat System tests", "-C metricbeat update system-tests-environment coverage-report") - } + makeTarget("Metricbeat System tests", "-C metricbeat update system-tests-environment coverage-report") } } - stage('Metricbeat oss'){ + stage('Metricbeat x-pack'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { beforeAgent true expression { - return env.BUILD_METRICBEAT != "false" + return env.BUILD_METRICBEAT_XPACK != "false" } } steps { - withBeatsEnv(){ - makeTarget("Metricbeat x-pack Linux", "-C x-pack/metricbeat testsuite") - } + makeTarget("Metricbeat x-pack Linux", "-C x-pack/metricbeat testsuite") } } stage('Metricbeat crosscompile'){ @@ -277,9 +298,7 @@ pipeline { } } steps { - withBeatsEnv(){ - makeTarget("Metricbeat oss crosscompile", "-C metricbeat crosscompile") - } + makeTarget("Metricbeat oss crosscompile", "-C metricbeat crosscompile") } } stage('Metricbeat Mac OS X'){ @@ -292,10 +311,22 @@ pipeline { } } steps { - withBeatsEnv(){ - makeTarget("Metricbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C metricbeat testsuite") + makeTarget("Metricbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C metricbeat testsuite") + } + } + stage('Metricbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && params.windowsTest } } + steps { + mageTargetWin("Metricbeat Windows Unit test", "-d metricbeat goUnitTest") + //mageTargetWin("Metricbeat Windows Integration test", "-d metricbeat goIntegTest") + } } stage('Packetbeat'){ agent { label 'ubuntu && immutable' } @@ -309,9 +340,7 @@ pipeline { stages { stage('Packetbeat oss'){ steps { - withBeatsEnv(){ - makeTarget("Packetbeat oss Linux", "-C packetbeat testsuite") - } + makeTarget("Packetbeat oss Linux", "-C packetbeat testsuite") } } } @@ -322,15 +351,13 @@ pipeline { when { beforeAgent true expression { - return env.BUILD_DOCKERLOGBEAT != "false" + return env.BUILD_DOCKERLOGBEAT_XPACK != "false" } } stages { stage('Dockerlogbeat'){ steps { - withBeatsEnv(){ - makeTarget("Elastic Log Plugin unit tests", "-C x-pack/dockerlogbeat testsuite") - } + makeTarget("Elastic Log Plugin unit tests", "-C x-pack/dockerlogbeat testsuite") } } } @@ -347,38 +374,71 @@ pipeline { stages { stage('Winlogbeat oss'){ steps { - withBeatsEnv(){ - makeTarget("Winlogbeat oss crosscompile", "-C winlogbeat crosscompile") + makeTarget("Winlogbeat oss crosscompile", "-C winlogbeat crosscompile") + } + } + stage('Winlogbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest } } + steps { + mageTargetWin("Winlogbeat Windows Unit test", "-d winlogbeat goUnitTest") + } } } } + stage('Winlogbeat Windows x-pack'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" + } + } + steps { + mageTargetWin("Winlogbeat Windows Unit test", "-d x-pack/winlogbeat update:fields goUnitTest") + } + } stage('Functionbeat'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { beforeAgent true expression { - return env.BUILD_FUNCTIONBEAT != "false" + return env.BUILD_FUNCTIONBEAT_XPACK != "false" } } stages { stage('Functionbeat x-pack'){ steps { - withBeatsEnv(){ - makeTarget("Functionbeat x-pack Linux", "-C x-pack/functionbeat testsuite") - } + makeTarget("Functionbeat x-pack Linux", "-C x-pack/functionbeat testsuite") } } stage('Functionbeat Mac OS X x-pack'){ agent { label 'macosx' } options { skipDefaultCheckout() } steps { - withBeatsEnv(){ - makeTarget("Functionbeat x-pack Mac OS X", "TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite") + makeTarget("Functionbeat x-pack Mac OS X", "TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite") + } + } + stage('Functionbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest } } + steps { + mageTargetWin("Functionbeat Windows Unit test", "-d x-pack/functionbeat goUnitTest") + } } } } @@ -394,9 +454,7 @@ pipeline { stages { stage('Journalbeat oss'){ steps { - withBeatsEnv(){ - makeTarget("Journalbeat Linux", "-C journalbeat testsuite") - } + makeTarget("Journalbeat Linux", "-C journalbeat testsuite") } } } @@ -413,34 +471,28 @@ pipeline { stages { stage('Generators Metricbeat Linux'){ steps { - withBeatsEnv(){ - makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test test-package") - } + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test") + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package") } } stage('Generators Beat Linux'){ steps { - withBeatsEnv(){ - makeTarget("Generators Beat Linux", "-C generator/_templates/beat test test-package") - } + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test") + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package") } } stage('Generators Metricbeat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } steps { - withBeatsEnv(){ - makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") - } + makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") } } stage('Generators Beat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } steps { - withBeatsEnv(){ - makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") - } + makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") } } } @@ -466,9 +518,7 @@ pipeline { expression { return env.BUILD_DOCS != "false" } } steps { - withBeatsEnv(){ - makeTarget("Docs", "docs") - } + makeTarget("Docs", "docs") } } } @@ -478,7 +528,26 @@ pipeline { def makeTarget(context, target, clean = true){ withGithubNotify(context: "${context}") { - sh(label: "Make ${target}", script: "make ${target}") + withBeatsEnv(){ + sh(label: "Make ${target}", script: """ + eval "\$(gvm use ${GO_VERSION} --format=bash)" + make ${target} + """) + if(clean) { + sh(script: 'script/fix_permissions.sh ${HOME}') + } + } + } +} + +def mageTargetWin(context, target){ + withGithubNotify(context: "${context}") { + withBeatsEnvWin(){ + bat(label: "Mage ${target}", script: """ + set + mage ${target} + """) + } } } @@ -486,47 +555,65 @@ def withBeatsEnv(Closure body){ withEnv([ "HOME=${env.WORKSPACE}", "GOPATH=${env.WORKSPACE}", - "PATH=${env.WORKSPACE}/bin:${env.PATH}", + "PATH+GO=${env.WORKSPACE}/bin:${env.PATH}", + "MAGEFILE_CACHE=${WORKSPACE}\\.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", ]){ deleteDir() unstash 'source' - sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") - sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") - - def envTmp = propertiesToEnv("go_env.properties") - withEnv(envTmp){ - dir("${BASE_DIR}"){ - try { - body() - } finally { - reportCoverage() - } + dir("${BASE_DIR}"){ + sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") + sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") + try { + body() + } finally { + reportCoverage() } } } } -def propertiesToEnv(file){ - def props = readProperties(file: file) - if(props.containsKey('PATH')){ - newPath = sh(label: 'eval path', returnStdout: true, script: "echo \"${props['PATH']}\"").trim() - props["PATH"] = "${newPath}:${env.PATH}" - } - def envTmp = [] - props.each { key, value -> - envTmp += "${key}=${value}" +def withBeatsEnvWin(Closure body){ + def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + withEnv([ + "HOME=${WORKSPACE}", + "GOPATH=${WORKSPACE}", + "PATH+GO=${WORKSPACE}\\bin;${goRoot}\\bin;C:\\ProgramData\\chocolatey\\bin", + "MAGEFILE_CACHE=${WORKSPACE}\\.magefile", + "GOROOT=${goRoot}", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + ]){ + deleteDir() + unstash 'source' + dir("${BASE_DIR}"){ + bat(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") + try { + body() + } finally { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') + } + } + } } - return envTmp } def k8sTest(versions){ versions.each{ v -> stage("k8s ${v}"){ withEnv(["K8S_VERSION=${v}"]){ - withBeatsEnv(){ - sh(label: "Install k8s", script: ".ci/scripts/kind-setup.sh") - makeTarget("Kubernetes Kind", "KUBECONFIG=\"\$(kind get kubeconfig-path)\" -C deploy/kubernetes test", false) - sh(label: 'Delete cluster', script: 'kind delete cluster') + withGithubNotify(context: "K8s ${v}") { + withBeatsEnv(){ + sh(label: "Install k8s", script: """ + eval "\$(gvm use ${GO_VERSION} --format=bash)" + .ci/scripts/kind-setup.sh + """) + sh(label: "Kubernetes Kind",script: "make KUBECONFIG=\"\$(kind get kubeconfig-path)\" -C deploy/kubernetes test") + sh(label: 'Delete cluster', script: 'kind delete cluster') + } } } } @@ -535,6 +622,8 @@ def k8sTest(versions){ def reportCoverage(){ catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**/TEST-*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/TEST-*.out') retry(2){ sh(label: 'Report to Codecov', script: ''' curl -sSLo codecov https://codecov.io/bash @@ -547,13 +636,80 @@ def reportCoverage(){ done ''') } - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**/TEST-*.xml") } } def isChanged(patterns){ - return (params.runAllStages + return ( + params.runAllStages || isGitRegionMatch(patterns: patterns, comparator: 'regexp') - || isGitRegionMatch(patterns: ["^libbeat/.*"], comparator: 'regexp') ) } + +def loadConfigEnvVars(){ + env.BUILD_AUDITBEAT = isChanged(["^auditbeat/.*"]) + env.BUILD_AUDITBEAT_XPACK = isChanged([ + "^auditbeat/.*", + "^x-pack/auditbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_DOCKERLOGBEAT_XPACK = isChanged([ + "^x-pack/dockerlogbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_DOCS = isChanged( + patterns: ["^docs/.*"], + comparator: 'regexp' + ) + env.BUILD_FILEBEAT = isChanged(["^filebeat/.*"]) + env.BUILD_FILEBEAT_XPACK = isChanged([ + "^filebeat/.*", + "^x-pack/filebeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_FUNCTIONBEAT_XPACK = isChanged([ + "^x-pack/functionbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_GENERATOR = isChanged(["^generator/.*"]) + env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"]) + env.BUILD_HEARTBEAT_XPACK = isChanged([ + "^heartbeat/.*", + "^x-pack/heartbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_JOURNALBEAT = isChanged(["^journalbeat/.*"]) + env.BUILD_JOURNALBEAT_XPACK = isChanged([ + "^journalbeat/.*", + "^x-pack/journalbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/*"]) + env.BUILD_LIBBEAT = isChanged( + patterns: ["^libbeat/.*"], + comparator: 'regexp' + ) + env.BUILD_LIBBEAT_XPACK = isChanged([ + "^libbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_METRICBEAT = isChanged(["^metricbeat/.*"]) + env.BUILD_METRICBEAT_XPACK = isChanged([ + "^metricbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/metricbeat/.*", + ]) + env.BUILD_PACKETBEAT = isChanged(["^packetbeat/.*"]) + env.BUILD_PACKETBEAT_XPACK = isChanged([ + "^packetbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/packetbeat/.*", + ]) + env.BUILD_WINLOGBEAT = isChanged(["^winlogbeat/.*"]) + env.BUILD_WINLOGBEAT_XPACK = isChanged([ + "^winlogbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/winlogbeat/.*", + ]) + env.GO_VERSION = readFile(".go-version").trim() +} diff --git a/NOTICE.txt b/NOTICE.txt index 5a849215e7a..0591c580b70 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1417,7 +1417,7 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-ucfg -Version: v0.8.1 +Version: v0.8.2 License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-ucfg/LICENSE: -------------------------------------------------------------------- diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index b285e5a2ed7..2fe13ec5e5d 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -511,6 +511,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -604,6 +610,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/dev-tools/make/mage.mk b/dev-tools/make/mage.mk index 76f507e3d67..bf8c443a5fe 100644 --- a/dev-tools/make/mage.mk +++ b/dev-tools/make/mage.mk @@ -7,7 +7,7 @@ export MAGE_IMPORT_PATH mage: ifndef MAGE_PRESENT @echo Installing mage $(MAGE_VERSION) from vendor dir. - @go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH} + go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH} @-mage -clean endif @true diff --git a/filebeat/Makefile b/filebeat/Makefile index 6e37e664325..940077b04cd 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -14,15 +14,15 @@ update: mage # Creates a new module. Requires the params MODULE. .PHONY: create-module -create-module: - @go run ${ES_BEATS}/filebeat/scripts/generator/module/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) +create-module: mage + mage generate:module # Creates a new fileset. Requires the params MODULE and FILESET. .PHONY: create-fileset -create-fileset: - @go run ${ES_BEATS}/filebeat/scripts/generator/fileset/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET) +create-fileset: mage + mage generate:fileset # Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET. .PHONY: create-fields -create-fields: - @go run ${ES_BEATS}/filebeat/scripts/generator/fields/main.go --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET) +create-fields: mage + mage generate:fields diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 701eded16af..73d08a67ca7 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -20140,6 +20140,304 @@ alias to: message -- +[float] +=== ingress_controller + +Contains fields for the Ingress Nginx controller access logs. + + + +*`nginx.ingress_controller.remote_ip_list`*:: ++ +-- +An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. + + +type: array + +-- + +*`nginx.ingress_controller.http.request.length`*:: ++ +-- +The request length (including request line, header, and request body) + + +type: long + +format: bytes + +-- + +*`nginx.ingress_controller.http.request.time`*:: ++ +-- +Time elapsed since the first bytes were read from the client + + +type: double + +format: duration + +-- + +*`nginx.ingress_controller.upstream.name`*:: ++ +-- +The name of the upstream. + + +type: text + +-- + +*`nginx.ingress_controller.upstream.alternative_name`*:: ++ +-- +The name of the alternative upstream. + + +type: text + +-- + +*`nginx.ingress_controller.upstream.response.length`*:: ++ +-- +The length of the response obtained from the upstream server + + +type: long + +format: bytes + +-- + +*`nginx.ingress_controller.upstream.response.time`*:: ++ +-- +The time spent on receiving the response from the upstream server as seconds with millisecond resolution + + +type: double + +format: duration + +-- + +*`nginx.ingress_controller.upstream.response.status_code`*:: ++ +-- +The status code of the response obtained from the upstream server + + +type: long + +-- + +*`nginx.ingress_controller.http.request.id`*:: ++ +-- +The randomly generated ID of the request + + +type: text + +-- + +*`nginx.ingress_controller.upstream.ip`*:: ++ +-- +The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas. + + +type: ip + +-- + +*`nginx.ingress_controller.upstream.port`*:: ++ +-- +The port of the upstream server. + + +type: long + +-- + +*`nginx.ingress_controller.body_sent.bytes`*:: ++ +-- +type: alias + +alias to: http.response.body.bytes + +-- + +*`nginx.ingress_controller.user_name`*:: ++ +-- +type: alias + +alias to: user.name + +-- + +*`nginx.ingress_controller.method`*:: ++ +-- +type: alias + +alias to: http.request.method + +-- + +*`nginx.ingress_controller.url`*:: ++ +-- +type: alias + +alias to: url.original + +-- + +*`nginx.ingress_controller.http_version`*:: ++ +-- +type: alias + +alias to: http.version + +-- + +*`nginx.ingress_controller.response_code`*:: ++ +-- +type: alias + +alias to: http.response.status_code + +-- + +*`nginx.ingress_controller.referrer`*:: ++ +-- +type: alias + +alias to: http.request.referrer + +-- + +*`nginx.ingress_controller.agent`*:: ++ +-- +type: alias + +alias to: user_agent.original + +-- + + +*`nginx.ingress_controller.user_agent.device`*:: ++ +-- +type: alias + +alias to: user_agent.device.name + +-- + +*`nginx.ingress_controller.user_agent.name`*:: ++ +-- +type: alias + +alias to: user_agent.name + +-- + +*`nginx.ingress_controller.user_agent.os`*:: ++ +-- +type: alias + +alias to: user_agent.os.full_name + +-- + +*`nginx.ingress_controller.user_agent.os_name`*:: ++ +-- +type: alias + +alias to: user_agent.os.name + +-- + +*`nginx.ingress_controller.user_agent.original`*:: ++ +-- +type: alias + +alias to: user_agent.original + +-- + + +*`nginx.ingress_controller.geoip.continent_name`*:: ++ +-- +type: alias + +alias to: source.geo.continent_name + +-- + +*`nginx.ingress_controller.geoip.country_iso_code`*:: ++ +-- +type: alias + +alias to: source.geo.country_iso_code + +-- + +*`nginx.ingress_controller.geoip.location`*:: ++ +-- +type: alias + +alias to: source.geo.location + +-- + +*`nginx.ingress_controller.geoip.region_name`*:: ++ +-- +type: alias + +alias to: source.geo.region_name + +-- + +*`nginx.ingress_controller.geoip.city_name`*:: ++ +-- +type: alias + +alias to: source.geo.city_name + +-- + +*`nginx.ingress_controller.geoip.region_iso_code`*:: ++ +-- +type: alias + +alias to: source.geo.region_iso_code + +-- + [[exported-fields-osquery]] == Osquery fields diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index e653f5de622..bc65b50bd2a 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -304,6 +304,14 @@ filebeat.modules: # can be added under this section. #input: + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + #ingress_controller: + # enabled: false + # + # # Set custom paths for the log files. If left empty, + # # Filebeat will choose the paths depending on your OS. + # #var.paths: + #------------------------------- Osquery Module ------------------------------- - module: osquery result: @@ -1208,6 +1216,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -1301,6 +1315,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/filebeat/fileset/modules_test.go b/filebeat/fileset/modules_test.go index d82f3361865..1c0d1766d36 100644 --- a/filebeat/fileset/modules_test.go +++ b/filebeat/fileset/modules_test.go @@ -56,7 +56,7 @@ func TestNewModuleRegistry(t *testing.T) { expectedModules := map[string][]string{ "auditd": {"log"}, - "nginx": {"access", "error"}, + "nginx": {"access", "error", "ingress_controller"}, "mysql": {"slowlog", "error"}, "system": {"syslog", "auth"}, } @@ -347,8 +347,8 @@ func TestMcfgFromConfig(t *testing.T) { Filesets: map[string]*FilesetConfig{ "error": { Enabled: &falseVar, - Var: map[string]interface{}{}, - Input: map[string]interface{}{}, + Var: nil, + Input: nil, }, }, }, @@ -366,7 +366,7 @@ func TestMcfgFromConfig(t *testing.T) { Var: map[string]interface{}{ "test": false, }, - Input: map[string]interface{}{}, + Input: nil, }, }, }, diff --git a/filebeat/magefile.go b/filebeat/magefile.go index d4ffea0ee45..e82403d361c 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -31,6 +31,8 @@ import ( // mage:import "github.com/elastic/beats/dev-tools/mage/target/common" + // mage:import generate + _ "github.com/elastic/beats/filebeat/scripts/mage/generate" ) func init() { diff --git a/filebeat/module/apache/access/ingest/default.json b/filebeat/module/apache/access/ingest/default.json deleted file mode 100644 index 6edd5b029be..00000000000 --- a/filebeat/module/apache/access/ingest/default.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", - "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", - "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", - "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})" - ], - "ignore_missing": true - } - }, - { - "remove": { - "field": "message" - } - }, - { - "grok": { - "field": "source.address", - "ignore_missing": true, - "patterns": [ - "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" - ] - } - }, - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "date": { - "field": "apache.access.time", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/yyyy:H:m:s Z" - ], - "ignore_failure": true - } - }, - { - "remove": { - "field": "apache.access.time", - "ignore_failure": true - } - }, - { - "user_agent": { - "field": "user_agent.original", - "ignore_failure": true - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml new file mode 100644 index 00000000000..ff905bd7245 --- /dev/null +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -0,0 +1,103 @@ +description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." + +processors: +- grok: + field: message + patterns: + - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "-" %{NUMBER:http.response.status_code:long} -' + - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original} + HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) + ignore_missing: true +- remove: + field: message +- set: + field: event.kind + value: event +- set: + field: event.category + value: web +- set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" +- set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" +- lowercase: + field: http.request.method + ignore_missing: true +- grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: apache.access.time + target_field: '@timestamp' + formats: + - dd/MMM/yyyy:H:m:s Z + ignore_failure: true +- remove: + field: apache.access.time + ignore_failure: true +- user_agent: + field: user_agent.original + ignore_failure: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: tls.cipher + value: '{{apache.access.ssl.cipher}}' + if: ctx?.apache?.access?.ssl?.cipher != null + +- script: + lang: painless + if: ctx?.apache?.access?.ssl?.protocol != null + source: >- + def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); + if (parts.length != 2) { + return; + } + if (parts[1].contains(".")) { + ctx.tls.version = parts[1]; + } else { + ctx.tls.version = parts[1] + ".0"; + } + ctx.tls.version_protocol = parts[0]; + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/apache/access/manifest.yml b/filebeat/module/apache/access/manifest.yml index 78f874c6330..ff7bcb0ca71 100644 --- a/filebeat/module/apache/access/manifest.yml +++ b/filebeat/module/apache/access/manifest.yml @@ -12,7 +12,7 @@ var: - "C:/tools/Apache/httpd-2.*/Apache24/logs/access.log*" - "C:/Program Files/Apache Software Foundation/Apache2.*/logs/access.log*" -ingest_pipeline: ingest/default.json +ingest_pipeline: ingest/pipeline.yml input: config/access.yml requires.processors: diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json index db1ab287374..4bf4ca896d6 100644 --- a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json @@ -1,10 +1,13 @@ [ { "@timestamp": "2016-12-26T14:16:28.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "success", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 45, "http.response.status_code": 200, "http.version": "1.1", @@ -18,10 +21,13 @@ }, { "@timestamp": "2016-12-26T14:16:29.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 209, "http.response.status_code": 404, "http.version": "1.1", @@ -35,8 +41,11 @@ }, { "@timestamp": "2016-12-26T14:16:48.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, "input.type": "log", @@ -48,10 +57,13 @@ }, { "@timestamp": "2016-12-26T16:23:35.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "success", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 45, "http.response.status_code": 200, "http.version": "1.1", @@ -74,10 +86,13 @@ }, { "@timestamp": "2016-12-26T16:23:41.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 206, "http.response.status_code": 404, "http.version": "1.1", @@ -100,10 +115,13 @@ }, { "@timestamp": "2016-12-26T16:23:45.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 201, "http.response.status_code": 404, "http.version": "1.1", diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index 8d2749b2416..946a3e22dab 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -3,10 +3,12 @@ "@timestamp": "2018-08-10T07:45:56.000Z", "apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "apache.access.ssl.protocol": "TLSv1.2", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 1375, "http.version": "1.1", "input.type": "log", @@ -14,16 +16,21 @@ "service.type": "apache", "source.address": "172.30.0.119", "source.ip": "172.30.0.119", + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" }, { "@timestamp": "2019-10-16T09:53:47.000Z", "apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "apache.access.ssl.protocol": "TLSv1.2", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.version": "1.1", "input.type": "log", "log.offset": 276, @@ -34,6 +41,9 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "11.19.0.217", + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json index a07d2244cc9..0a593646626 100644 --- a/filebeat/module/apache/access/test/test-vhost.log-expected.json +++ b/filebeat/module/apache/access/test/test-vhost.log-expected.json @@ -2,10 +2,13 @@ { "@timestamp": "2016-12-26T16:22:14.000Z", "destination.domain": "vhost1.domaine.fr", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index bcbf37f2a09..0c1520846fb 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -1,10 +1,13 @@ [ { "@timestamp": "2016-12-26T14:16:29.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.response.body.bytes": 209, "http.response.status_code": 404, "http.version": "1.1", @@ -18,10 +21,13 @@ }, { "@timestamp": "2016-12-26T16:22:13.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, @@ -43,8 +49,11 @@ }, { "@timestamp": "2016-12-26T14:16:48.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, "input.type": "log", @@ -56,10 +65,13 @@ }, { "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 612, "http.response.status_code": 404, @@ -81,10 +93,13 @@ }, { "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "success", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 612, "http.response.status_code": 200, @@ -106,8 +121,11 @@ }, { "@timestamp": "2019-02-02T04:38:45.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", "http.request.referrer": "-", "http.response.body.bytes": 152, diff --git a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json index 7cc249242ef..2fbd7b9ffb6 100644 --- a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json @@ -1,10 +1,13 @@ [ { "@timestamp": "2016-12-26T16:18:09.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "success", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 491, "http.response.status_code": 200, @@ -24,10 +27,13 @@ }, { "@timestamp": "2016-12-26T16:22:00.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "success", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 484, "http.response.status_code": 200, @@ -49,10 +55,13 @@ }, { "@timestamp": "2016-12-26T16:22:00.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://192.168.33.72/", "http.response.body.bytes": 504, "http.response.status_code": 404, @@ -74,10 +83,13 @@ }, { "@timestamp": "2016-12-26T16:22:08.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "success", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 484, "http.response.status_code": 200, @@ -99,10 +111,13 @@ }, { "@timestamp": "2016-12-26T16:22:08.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 504, "http.response.status_code": 404, @@ -124,10 +139,13 @@ }, { "@timestamp": "2016-12-26T16:22:08.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 504, "http.response.status_code": 404, @@ -149,10 +167,13 @@ }, { "@timestamp": "2016-12-26T16:22:10.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 498, "http.response.status_code": 404, @@ -174,10 +195,13 @@ }, { "@timestamp": "2016-12-26T16:22:13.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, @@ -199,10 +223,13 @@ }, { "@timestamp": "2016-12-26T16:22:17.000Z", + "event.category": "web", "event.dataset": "apache.access", + "event.kind": "event", "event.module": "apache", + "event.outcome": "failure", "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "-", "http.response.body.bytes": 499, "http.response.status_code": 404, diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json deleted file mode 100644 index 89a6624aea9..00000000000 --- a/filebeat/module/apache/error/ingest/pipeline.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "description": "Pipeline for parsing apache error logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}", - "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}" - ], - "pattern_definitions": { - "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" - }, - "ignore_missing": true - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "apache.error.timestamp", - "target_field": "@timestamp", - "formats": [ - "EEE MMM dd H:m:s yyyy", - "EEE MMM dd H:m:s.SSSSSS yyyy" - ], - "on_failure": [ - { - "append": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "apache.error.timestamp", - "target_field": "@timestamp", - "formats": [ - "EEE MMM dd H:m:s yyyy", - "EEE MMM dd H:m:s.SSSSSS yyyy" - ], - "timezone": "{{ event.timezone }}", - "on_failure": [ - { - "append": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] - } - }, - { - "remove": { - "field": "apache.error.timestamp", - "ignore_failure": true - } - }, - { - "grok": { - "field": "source.address", - "ignore_missing": true, - "patterns": [ - "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" - ] - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml new file mode 100644 index 00000000000..967f7a34b69 --- /dev/null +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -0,0 +1,85 @@ +description: Pipeline for parsing apache error logs +processors: +- grok: + field: message + patterns: + - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client + %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} + - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\] + \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( + \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} + pattern_definitions: + APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' + ignore_missing: true +- date: + if: ctx.event.timezone == null + field: apache.error.timestamp + target_field: '@timestamp' + formats: + - EEE MMM dd H:m:s yyyy + - EEE MMM dd H:m:s.SSSSSS yyyy + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: apache.error.timestamp + target_field: '@timestamp' + formats: + - EEE MMM dd H:m:s yyyy + - EEE MMM dd H:m:s.SSSSSS yyyy + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- remove: + field: apache.error.timestamp + ignore_failure: true +- set: + field: event.kind + value: event +- set: + field: event.category + value: web +- script: + if: "ctx?.log?.level != null" + lang: painless + source: >- + def err_levels = ["emerg", "alert", "crit", "error", "warn"]; + if (err_levels.contains(ctx.log.level)) { + ctx.event.type = "error"; + } else { + ctx.event.type = "info"; + } + +- grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/apache/error/manifest.yml b/filebeat/module/apache/error/manifest.yml index 7c60dc4089e..5b677ef2cdd 100644 --- a/filebeat/module/apache/error/manifest.yml +++ b/filebeat/module/apache/error/manifest.yml @@ -11,5 +11,5 @@ var: - "C:/tools/Apache/httpd-2.*/Apache24/logs/error.log*" - "C:/Program Files/Apache Software Foundation/Apache2.*/logs/error.log*" -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/error.yml diff --git a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json index 004e23a45f4..4453f2bebec 100644 --- a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json @@ -2,9 +2,12 @@ { "@timestamp": "2016-12-26T16:15:55.103-02:00", "apache.error.module": "mpm_prefork", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "error", "input.type": "log", "log.level": "notice", @@ -16,9 +19,12 @@ { "@timestamp": "2016-12-26T16:15:55.103-02:00", "apache.error.module": "core", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "error", "input.type": "log", "log.level": "notice", diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json index eab41ed8f4d..fbc9605ef38 100644 --- a/filebeat/module/apache/error/test/test.log-expected.json +++ b/filebeat/module/apache/error/test/test.log-expected.json @@ -1,9 +1,12 @@ [ { "@timestamp": "2016-12-26T16:22:08.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -16,9 +19,12 @@ { "@timestamp": "2016-12-26T16:15:55.103-02:00", "apache.error.module": "core", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "error", "input.type": "log", "log.level": "notice", @@ -30,9 +36,12 @@ { "@timestamp": "2011-09-09T10:42:29.902-02:00", "apache.error.module": "core", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -56,9 +65,12 @@ { "@timestamp": "2019-06-27T06:58:09.169-02:00", "apache.error.module": "include", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "warn", diff --git a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json index 6133c89aa65..41adf5d7ae5 100644 --- a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json @@ -1,9 +1,12 @@ [ { "@timestamp": "2016-12-26T16:17:53.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "error", "input.type": "log", "log.level": "notice", @@ -13,9 +16,12 @@ }, { "@timestamp": "2016-12-26T16:22:00.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -27,9 +33,12 @@ }, { "@timestamp": "2016-12-26T16:22:08.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -41,9 +50,12 @@ }, { "@timestamp": "2016-12-26T16:22:08.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -55,9 +67,12 @@ }, { "@timestamp": "2016-12-26T16:22:10.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -69,9 +84,12 @@ }, { "@timestamp": "2016-12-26T16:22:13.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -83,9 +101,12 @@ }, { "@timestamp": "2016-12-26T16:22:17.000-02:00", + "event.category": "web", "event.dataset": "apache.error", + "event.kind": "event", "event.module": "apache", "event.timezone": "-02:00", + "event.type": "error", "fileset.name": "error", "input.type": "log", "log.level": "error", diff --git a/filebeat/module/nginx/_meta/config.reference.yml b/filebeat/module/nginx/_meta/config.reference.yml index 572341217e6..cdfe3e99be7 100644 --- a/filebeat/module/nginx/_meta/config.reference.yml +++ b/filebeat/module/nginx/_meta/config.reference.yml @@ -22,3 +22,11 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + #ingress_controller: + # enabled: false + # + # # Set custom paths for the log files. If left empty, + # # Filebeat will choose the paths depending on your OS. + # #var.paths: diff --git a/filebeat/module/nginx/_meta/config.yml b/filebeat/module/nginx/_meta/config.yml index 22fe90e486e..3967af2693f 100644 --- a/filebeat/module/nginx/_meta/config.yml +++ b/filebeat/module/nginx/_meta/config.yml @@ -14,3 +14,11 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + ingress_controller: + enabled: false + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/module/nginx/fields.go b/filebeat/module/nginx/fields.go index f57f8e6d2a4..e45823e11eb 100644 --- a/filebeat/module/nginx/fields.go +++ b/filebeat/module/nginx/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNginx returns asset data. // This is the base64 encoded gzipped contents of module/nginx. func AssetNginx() string { - return "eJy0l8Fu4zgMhu95CqLn1g+QwwKLAgV62MViT3tzFYl2iMqil6Q7k7cf2Enb1JETO+nolNrm/38UKVV6gFfcrSHVlH6uAIws4hru/u7/vlsBBFQv1BpxWsMfKwCAvzh0EaFigdaJUqrBtghDCESuoaKIWqwAdMtipedUUb0Gkw5XABVhDLoepB4guQY/7fthuxbXUAt37eFJhqEfT4MQVMLNFEA/jv2OPZ33qPrxOGd8xrwfj5zMUdKDxTAjnyB7/Z7nAyWHc4wk2LBhSW0ZSe3LJ+94TsTtRm/OIPbjz7SPAq4ODvD8D7gQBFVRC3g2IAUHvSls0LtOEWh46LlpOIExUPKxC3gPG1QKqEOmPhKmMSgcyd9/sdrXaosuoChEekV4+e/hieWHk4Ch//VSnKj9iy6Ccid+ACcFQTUWDD3Xy/5NQe1RaHZ2Nxx2pWKyYrMz1Pz0RnLjN62z7Rq2Zm0hqC0nxaLXyso0VIvbV+LQ76cgnaKU/c+FCH1ckYmb49mgbTlcl/P/HaoVWYVZ6UpcmqjEgoVqSm4cOsewxy7fUJQ4XZNxPnSO83t/lJ7D0up+bTA1Z53mdOZxVCiCcku9JzTm2Lv6dFeY09zlELi09NNrbJpjvMvDxM58LBnwjfy4GudTy6a318kt5HNZjmkmwhey3AjBOZOFCKxF1cWY2xCXoUwpLOe5FSXfv9ewTCstWg01Mo3b/fqF4DkZJUx2y4wf/m3XyMVFvbkT77lLJruSlHM751VoFxXnwkX2w2e3Q51RmgsjWBOnb6rfebHZxSPbfVdDnZFaOEPf10qXBafQ3pFQhOX3XVgG+UX3Fc8pobchrfzZMnKql91WHj80gQImo4pQLpzsI77h0jNm5LrIxc0537QT2U67tcL9bbA4jZzjZ1f72VbQheIq1wZVXb30GJuPGvv9CgAA//9Q+TWn" + return "eJzsWM2O4zYMvs9TENtLC8z4AXIoUGyxwBxaFEUPvWUVi3aIlUWXlDObty9kOz/jkRM7md32YJ9m7PD7PooUJfIJvuB+Bb4k//UBIFBwuIIPv8f/PzwAWNRcqA7EfgU/PwAA/Ma2cQgFC9RGlHwJYYvQmoDjEgpyqNkDgG5ZwjpnX1C5giANPgAUhM7qqoV6Am8qPNHHJ+xrXEEp3NT9m4SG+HxqgaAQrsYExOec75zT5DmqHl+niC+Qx+cj+2DIa0/RrshJSIcf9RylpOScSxKsOOCa6rUjDa9+cpBnRMx+8OWCxPj84jsr4KJngOc/wFgrqIqawXMAUjAQSWGDuWkUgdqXOVcVewgM5HPXWHyEDSpZ1NbT3BH6oVA4g398RdXFaovGoig4+oLw+e+nTywvRiza+Nfn7A3an2gcKDeSt8JJQVADC9qo63P3JaP6zDS5uhu2+7WiD9lmH1DTy+vIDL/UJmxXsA2hzgS1Zq+YRawkTEWlmC4Sfb6/FdIoyjr+OVNCtMsSdlM4Kwxbtrf5/E+DGrIkwiR3xc11VFzGQiV5MzSdQhhlr3coSuxv8ThtOoX5kB/rnO3c6L5OMA0mNJrCmaajQBGUe+I9gjGF3pRvq8KU5F63hnNDP77HxnUMqzyMVOZzSIs7yofRuOxa0r0OJ7WRL3k5VDNiPlPLnSI4RTJTAmtWNM6lCuI8KWMI8/XcKyWdv7doGUeatRtKZBqm++0bIWcfyKMP96x4f2yXyNlVvKkLn3Pjg+zXpJyqnDdJu4o4VZzjvP3Z/aIuIE0VI1gS+3eK32WwycGjsH+vhLoANXOF3i+VrgOOSTtIQhGWb9ewtPCz+pWcvcc8tG6l75aOfTmvW/l4xASy6AMVhHLlZu9wh3PvmI7LLGU35X5Tj3g7zlYLx24we2s5hS/czBe2gsZmN7FWqGrKudfYtNW11CZfxhZxHY8CYefwm+T5c8fS5/uJa+nV/y+9+g+HTHg6i47WmFNBeR+P0Z7v2Lo49GXYTi1IBUtlwgpS7fyVcP21RehJoSOFH7vFJ1+evpDHx34pH8F4e/yyYbv/6bo/gUZmBZabjRt+OvhjG0ldE665RBUCOlMrWlDyObbZU5BEvXGJ4AUlum3safaWSK5jJ1ZrEDRV6lbdeRHw6zAvJ6x7hIvbJNIfOS4rMC6geBNoh+Pzl/vVnNFMVXbs+79r6vYp28s+aADexOqJZ+E96ARF2Q1GAuO+fNe03SJEvlgsfAD2IJgj7Q7T6aNzYz6BUVDM2VuFFwpbqMg56t5Ea3bNG1Xjro+Pb269F0UHO1SIqO8UtFdlZuSWcduGEOMtV24PJXoUE9DC868n1S3j5cUc6VjfvJ4g5nSwDStGvzgZPBeguEOJ51f7qq9z8RwyeZRvGzmv6v0Vi3z5GBFJzs5OIwiKten83uzbs9nolSJQs6QvErdlS4Qbc/fyfXqZlC+T8mVSvkzKl0n5Mim/rGeZlC+T8mVSvkzK/+NJ+b8BAAD//1+3mU8=" } diff --git a/filebeat/module/nginx/ingress_controller/_meta/fields.yml b/filebeat/module/nginx/ingress_controller/_meta/fields.yml new file mode 100644 index 00000000000..0c9ca13de32 --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/_meta/fields.yml @@ -0,0 +1,142 @@ +- name: ingress_controller + type: group + description: > + Contains fields for the Ingress Nginx controller access logs. + fields: + - name: remote_ip_list + type: array + description: > + An array of remote IP addresses. It is a list because it is common to include, besides the client + IP address, IP addresses from headers like `X-Forwarded-For`. + Real source IP is restored to `source.ip`. + + # ingress-controller specific fields + - name: http.request.length + type: long + format: bytes + description: > + The request length (including request line, header, and request body) + - name: http.request.time + type: double + format: duration + description: > + Time elapsed since the first bytes were read from the client + - name: upstream.name + type: text + description: > + The name of the upstream. + - name: upstream.alternative_name + type: text + description: > + The name of the alternative upstream. + - name: upstream.response.length + type: long + format: bytes + description: > + The length of the response obtained from the upstream server + - name: upstream.response.time + type: double + format: duration + description: > + The time spent on receiving the response from the upstream server as seconds with millisecond resolution + - name: upstream.response.status_code + type: long + description: > + The status code of the response obtained from the upstream server + - name: http.request.id + type: text + description: > + The randomly generated ID of the request + - name: upstream.ip + type: ip + description: > + The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas. + - name: upstream.port + type: long + description: > + The port of the upstream server. + + - name: body_sent.bytes + type: alias + path: http.response.body.bytes + migration: true + - name: user_name + type: alias + path: user.name + migration: true + - name: method + type: alias + path: http.request.method + migration: true + - name: url + type: alias + path: url.original + migration: true + - name: http_version + type: alias + path: http.version + migration: true + - name: response_code + type: alias + path: http.response.status_code + migration: true + - name: referrer + type: alias + path: http.request.referrer + migration: true + - name: agent + type: alias + path: user_agent.original + migration: true + + - name: user_agent + type: group + fields: + - name: device + type: alias + path: user_agent.device.name + migration: true + - name: name + type: alias + path: user_agent.name + migration: true + - name: os + type: alias + path: user_agent.os.full_name + migration: true + - name: os_name + type: alias + path: user_agent.os.name + migration: true + - name: original + type: alias + path: user_agent.original + migration: true + + - name: geoip + type: group + fields: + - name: continent_name + type: alias + path: source.geo.continent_name + migration: true + - name: country_iso_code + type: alias + path: source.geo.country_iso_code + migration: true + - name: location + type: alias + path: source.geo.location + migration: true + - name: region_name + type: alias + path: source.geo.region_name + migration: true + - name: city_name + type: alias + path: source.geo.city_name + migration: true + - name: region_iso_code + type: alias + path: source.geo.region_iso_code + migration: true diff --git a/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml b/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml new file mode 100644 index 00000000000..63db16170d4 --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml @@ -0,0 +1,9 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +processors: + - add_locale: ~ diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json new file mode 100644 index 00000000000..e660f22f022 --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json @@ -0,0 +1,151 @@ +{ + "description": "Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins.", + "processors": [ + { + "grok": { + "field": "message", + "patterns": [ + "(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - %{DATA:user.name} \\[%{HTTPDATE:nginx.ingress_controller.time}\\] \"%{DATA:nginx.ingress_controller.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:user_agent.original}\" %{NUMBER:nginx.ingress_controller.http.request.length:long} %{NUMBER:nginx.ingress_controller.http.request.time:double} \\[%{DATA:nginx.ingress_controller.upstream.name}\\] \\[%{DATA:nginx.ingress_controller.upstream.alternative_name}\\] (%{UPSTREAM_ADDRESS}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.length:long}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.time:double}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.status_code:long}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id}" + ], + "pattern_definitions": { + "NGINX_HOST": "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?", + "NGINX_NOTSEPARATOR": "[^\t ,:]+", + "NGINX_ADDRESS_LIST": "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*", + "UPSTREAM_ADDRESS": "%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?" + }, + "ignore_missing": true + } + }, + { + "grok": { + "field": "nginx.ingress_controller.info", + "patterns": [ + "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", + "" + ], + "ignore_missing": true + } + }, + { + "remove": { + "field": "nginx.ingress_controller.info" + } + }, + { + "split": { + "field": "nginx.ingress_controller.remote_ip_list", + "separator": "\"?,?\\s+", + "ignore_missing": true + } + }, + { + "split": { + "field": "nginx.ingress_controller.origin", + "separator": "\"?,?\\s+", + "ignore_missing": true + } + }, + { + "set": { + "field": "source.address", + "if": "ctx.source?.address == null", + "value": "" + } + }, + { + "script": { + "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0", + "lang": "painless", + "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } try { ctx.source.address = null; if (ctx.nginx.ingress_controller.remote_ip_list == null) { return; } def found = false; for (def item : ctx.nginx.ingress_controller.remote_ip_list) { if (!isPrivate(params.dot, item)) { ctx.source.address = item; found = true; break; } } if (!found) { ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; }} catch (Exception e) { ctx.source.address = null; }", + "params": { + "dot": "." + } + } + }, + { + "remove": { + "field": "source.address", + "if": "ctx.source.address == null" + } + }, + { + "grok": { + "field": "source.address", + "patterns": ["^%{IP:source.ip}$"], + "ignore_failure": true + } + }, + { + "remove": { + "field": "message" + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "date": { + "field": "nginx.ingress_controller.time", + "target_field": "@timestamp", + "formats": [ + "dd/MMM/yyyy:H:m:s Z" + ], + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] + } + }, + { + "remove": { + "field": "nginx.ingress_controller.time" + } + }, + { + "user_agent": { + "field": "user_agent.original" + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/filebeat/module/nginx/ingress_controller/manifest.yml b/filebeat/module/nginx/ingress_controller/manifest.yml new file mode 100644 index 00000000000..0f51e4d5c04 --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/nginx/access.log* + os.darwin: + - /usr/local/var/log/nginx/access.log* + os.windows: + - c:/programdata/nginx/logs/*access.log* + +ingest_pipeline: ingest/pipeline.json +input: config/ingress_controller.yml + +requires.processors: + - name: user_agent + plugin: ingest-user-agent + - name: geoip + plugin: ingest-geoip diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log new file mode 100644 index 00000000000..862c03a4af2 --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -0,0 +1,22 @@ +192.168.64.1 - - [07/Feb/2020:11:48:51 +0000] "POST /products HTTP/1.1" 200 59 "-" "curl/7.54.0" 89 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 529a007902362a5f51385a5fa7049884 +192.168.64.1 - - [07/Feb/2020:11:49:15 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "curl/7.54.0" 91 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 68fa971ce4dfce685fdc01c877bfa645 +192.168.64.1 - - [07/Feb/2020:11:49:30 +0000] "DELETE /products/42 HTTP/1.1" 200 59 "-" "curl/7.54.0" 94 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 0be411044cb1cb67580e115413b2da60 +192.168.64.1 - - [07/Feb/2020:11:49:43 +0000] "PATCH /products/42 HTTP/1.1" 200 59 "-" "curl/7.54.0" 93 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 f479ab1d9cc8afcbac9e9f958ff8babc +192.168.64.1 - - [07/Feb/2020:11:49:50 +0000] "PATCHp /products/42 HTTP/1.1" 400 163 "-" "-" 0 0.000 [] [] - - - - 4c7d2079340e68353c7d0dfff00b904b +192.168.64.1 - - [07/Feb/2020:11:50:09 +0000] "geti /products/42 HTTP/1.1" 400 163 "-" "-" 0 0.000 [] [] - - - - efb0c5aa8be6cdeb4a7e7bd090e3d893 +192.168.64.1 - - [07/Feb/2020:11:55:05 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Wget/1.20.3 (darwin18.6.0)" 157 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 457b71c3e1ee1887bb809effd301a0ec +192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 450 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 da29abf31e4d6324cebe5e7bca370709 +192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/products/42" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 381 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 e983c8cf3d713548baa50c9e2fffeb34 +192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 441 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 3d7ff18ff4181a7db5013a76f975d900 +192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/v2" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 372 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 d131fe4bcd359cf947f75efca4bfa553 +192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 369 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 ef6629fcaaa1ea0d1a843cf2bf40571d +192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/products/42" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 325 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 2593a1126588922449c183c8d9ddbbeb +192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 369 0.002 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 0f76ea730f282d5759018eb756b23b14 +192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET / HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 358 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 21efd18e3a7952fc78c0f2dcc1f05e69 +192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 314 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 e096809c2cb46f004c3b538b23916e5b +192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 360 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.002 200 0a2a92d080e664dd4e95c85d097c9d3d +192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/v2" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 316 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 4c024ebfc20acfb2d59e542e3ed60789 +192.168.64.1 - - [07/Feb/2020:12:00:28 +0000] "GET /products/42?address=delhi+technological+university HTTP/1.1" 200 59 "-" "python-requests/2.22.0" 197 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 9a7babf34ca4ee59d90ac48d452a9214 +192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79 +192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f +192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json new file mode 100644 index 00000000000..2dc9d1afbce --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -0,0 +1,802 @@ +[ + { + "@timestamp": "2020-02-07T11:48:51.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "POST", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 0, + "nginx.ingress_controller.http.request.id": "529a007902362a5f51385a5fa7049884", + "nginx.ingress_controller.http.request.length": 89, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/products", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.54.0", + "user_agent.version": "7.54.0" + }, + { + "@timestamp": "2020-02-07T11:49:15.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 190, + "nginx.ingress_controller.http.request.id": "68fa971ce4dfce685fdc01c877bfa645", + "nginx.ingress_controller.http.request.length": 91, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.54.0", + "user_agent.version": "7.54.0" + }, + { + "@timestamp": "2020-02-07T11:49:30.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "DELETE", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 382, + "nginx.ingress_controller.http.request.id": "0be411044cb1cb67580e115413b2da60", + "nginx.ingress_controller.http.request.length": 94, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.54.0", + "user_agent.version": "7.54.0" + }, + { + "@timestamp": "2020-02-07T11:49:43.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "PATCH", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 577, + "nginx.ingress_controller.http.request.id": "f479ab1d9cc8afcbac9e9f958ff8babc", + "nginx.ingress_controller.http.request.length": 93, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.54.0", + "user_agent.version": "7.54.0" + }, + { + "@timestamp": "2020-02-07T11:49:50.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "PATCHp", + "http.request.referrer": "-", + "http.response.body.bytes": 163, + "http.response.status_code": 400, + "http.version": "1.1", + "input.type": "log", + "log.offset": 771, + "nginx.ingress_controller.http.request.id": "4c7d2079340e68353c7d0dfff00b904b", + "nginx.ingress_controller.http.request.length": 0, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.name": "", + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "-" + }, + { + "@timestamp": "2020-02-07T11:50:09.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "geti", + "http.request.referrer": "-", + "http.response.body.bytes": 163, + "http.response.status_code": 400, + "http.version": "1.1", + "input.type": "log", + "log.offset": 919, + "nginx.ingress_controller.http.request.id": "efb0c5aa8be6cdeb4a7e7bd090e3d893", + "nginx.ingress_controller.http.request.length": 0, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.name": "", + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "-" + }, + { + "@timestamp": "2020-02-07T11:55:05.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1065, + "nginx.ingress_controller.http.request.id": "457b71c3e1ee1887bb809effd301a0ec", + "nginx.ingress_controller.http.request.length": 157, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Wget", + "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", + "user_agent.version": "1.20.3" + }, + { + "@timestamp": "2020-02-07T11:55:57.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1273, + "nginx.ingress_controller.http.request.id": "da29abf31e4d6324cebe5e7bca370709", + "nginx.ingress_controller.http.request.length": 450, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "79.0.3945.130" + }, + { + "@timestamp": "2020-02-07T11:55:57.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "http://hello-world.info/products/42", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1576, + "nginx.ingress_controller.http.request.id": "e983c8cf3d713548baa50c9e2fffeb34", + "nginx.ingress_controller.http.request.length": 381, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/favicon.ico", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "79.0.3945.130" + }, + { + "@timestamp": "2020-02-07T11:56:24.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1913, + "nginx.ingress_controller.http.request.id": "3d7ff18ff4181a7db5013a76f975d900", + "nginx.ingress_controller.http.request.length": 441, + "nginx.ingress_controller.http.request.time": 0.002, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/v2", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "79.0.3945.130" + }, + { + "@timestamp": "2020-02-07T11:56:24.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "http://hello-world.info/v2", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2208, + "nginx.ingress_controller.http.request.id": "d131fe4bcd359cf947f75efca4bfa553", + "nginx.ingress_controller.http.request.length": 372, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.002, + "service.type": "nginx", + "source.address": "", + "url.original": "/favicon.ico", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "79.0.3945.130" + }, + { + "@timestamp": "2020-02-07T11:56:36.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2536, + "nginx.ingress_controller.http.request.id": "ef6629fcaaa1ea0d1a843cf2bf40571d", + "nginx.ingress_controller.http.request.length": 369, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T11:56:36.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "http://hello-world.info/products/42", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2837, + "nginx.ingress_controller.http.request.id": "2593a1126588922449c183c8d9ddbbeb", + "nginx.ingress_controller.http.request.length": 325, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/favicon.ico", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T11:56:54.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 3172, + "nginx.ingress_controller.http.request.id": "0f76ea730f282d5759018eb756b23b14", + "nginx.ingress_controller.http.request.length": 369, + "nginx.ingress_controller.http.request.time": 0.002, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.002, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T11:56:54.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 3473, + "nginx.ingress_controller.http.request.id": "21efd18e3a7952fc78c0f2dcc1f05e69", + "nginx.ingress_controller.http.request.length": 358, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T11:56:54.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "http://hello-world.info/", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 3763, + "nginx.ingress_controller.http.request.id": "e096809c2cb46f004c3b538b23916e5b", + "nginx.ingress_controller.http.request.length": 314, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.002, + "service.type": "nginx", + "source.address": "", + "url.original": "/favicon.ico", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T11:56:56.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4087, + "nginx.ingress_controller.http.request.id": "0a2a92d080e664dd4e95c85d097c9d3d", + "nginx.ingress_controller.http.request.length": 360, + "nginx.ingress_controller.http.request.time": 0.002, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.002, + "service.type": "nginx", + "source.address": "", + "url.original": "/v2", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T11:56:56.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "http://hello-world.info/v2", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4380, + "nginx.ingress_controller.http.request.id": "4c024ebfc20acfb2d59e542e3ed60789", + "nginx.ingress_controller.http.request.length": 316, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/favicon.ico", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" + }, + { + "@timestamp": "2020-02-07T12:00:28.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4706, + "nginx.ingress_controller.http.request.id": "9a7babf34ca4ee59d90ac48d452a9214", + "nginx.ingress_controller.http.request.length": 197, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/products/42?address=delhi+technological+university", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Python Requests", + "user_agent.original": "python-requests/2.22.0", + "user_agent.version": "2.22" + }, + { + "@timestamp": "2020-02-07T12:02:38.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4949, + "nginx.ingress_controller.http.request.id": "ba91c30454893c121879396b0a78be79", + "nginx.ingress_controller.http.request.length": 343, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.001, + "service.type": "nginx", + "source.address": "", + "url.original": "/v2", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + }, + { + "@timestamp": "2020-02-07T12:02:38.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 5205, + "nginx.ingress_controller.http.request.id": "98c81aa2d50c67f6fb1fa16d5ce62f8f", + "nginx.ingress_controller.http.request.length": 262, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.5", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/favicon.ico", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + }, + { + "@timestamp": "2020-02-07T12:02:42.000Z", + "event.dataset": "nginx.ingress_controller", + "event.module": "nginx", + "event.timezone": "-02:00", + "fileset.name": "ingress_controller", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 5469, + "nginx.ingress_controller.http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "nginx.ingress_controller.http.request.length": 348, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.6", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": "8080", + "nginx.ingress_controller.upstream.response.length": 61, + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.time": 0.0, + "service.type": "nginx", + "source.address": "", + "url.original": "/v2/some", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + } +] \ No newline at end of file diff --git a/filebeat/module/system/auth/test/timestamp.log-expected.json b/filebeat/module/system/auth/test/timestamp.log-expected.json index 2b881dc5fa0..80c07d4e9a8 100644 --- a/filebeat/module/system/auth/test/timestamp.log-expected.json +++ b/filebeat/module/system/auth/test/timestamp.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2019-06-14T08:40:20.912-02:00", + "@timestamp": "2019-06-14T10:40:20.912-02:00", "event.dataset": "system.auth", "event.module": "system", "event.timezone": "-02:00", diff --git a/filebeat/module/system/syslog/test/tz-offset.log-expected.json b/filebeat/module/system/syslog/test/tz-offset.log-expected.json index 0e9c7d984cc..154e256b2ba 100644 --- a/filebeat/module/system/syslog/test/tz-offset.log-expected.json +++ b/filebeat/module/system/syslog/test/tz-offset.log-expected.json @@ -29,7 +29,7 @@ "service.type": "system" }, { - "@timestamp": "2019-06-14T08:40:20.912-02:00", + "@timestamp": "2019-06-14T10:40:20.912-02:00", "event.dataset": "system.syslog", "event.module": "system", "event.timezone": "-02:00", diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled index cc77a6b3459..e15f4fe492d 100644 --- a/filebeat/modules.d/nginx.yml.disabled +++ b/filebeat/modules.d/nginx.yml.disabled @@ -17,3 +17,11 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + ingress_controller: + enabled: false + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/scripts/generator/fileset/main.go b/filebeat/scripts/generator/fileset/main.go deleted file mode 100644 index 320ddbc55ef..00000000000 --- a/filebeat/scripts/generator/fileset/main.go +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/elastic/beats/filebeat/generator/fileset" -) - -func main() { - module := flag.String("module", "", "Name of the module") - filesetName := flag.String("fileset", "", "Name of the fileset") - modulesPath := flag.String("path", ".", "Path to the generated fileset") - beatsPath := flag.String("beats_path", ".", "Path to elastic/beats") - flag.Parse() - - if *module == "" { - fmt.Println("Missing parameter: module") - os.Exit(1) - } - - if *filesetName == "" { - fmt.Println("Missing parameter: fileset") - os.Exit(1) - } - - err := fileset.Generate(*module, *filesetName, *modulesPath, *beatsPath) - if err != nil { - fmt.Printf("Cannot generate fileset: %v\n", err) - os.Exit(3) - } - - fmt.Println("New fileset was generated, please check that module.yml file have proper fileset dashboard settings. After setting up Grok pattern in pipeline.json, please generate fields.yml") -} diff --git a/filebeat/scripts/mage/generate/fields.go b/filebeat/scripts/mage/generate/fields.go new file mode 100644 index 00000000000..9bc8a8af40a --- /dev/null +++ b/filebeat/scripts/mage/generate/fields.go @@ -0,0 +1,42 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package generate + +import ( + "fmt" + "os" + + devtools "github.com/elastic/beats/dev-tools/mage" + genfields "github.com/elastic/beats/filebeat/generator/fields" +) + +// Fields creates a new fields.yml for an existing Filebeat fileset. +// Use MODULE=module to specify the name of the existing module +// Use FILESET=fileset to specify the name of the existing fileset +func Fields() error { + targetModule := os.Getenv("MODULE") + targetFileset := os.Getenv("FILESET") + + if targetModule == "" || targetFileset == "" { + return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset mage generate:fields") + } + + curDir := devtools.CWD() + + return genfields.Generate(curDir, targetModule, targetFileset, false) +} diff --git a/filebeat/scripts/mage/generate/fileset.go b/filebeat/scripts/mage/generate/fileset.go new file mode 100644 index 00000000000..612567a1104 --- /dev/null +++ b/filebeat/scripts/mage/generate/fileset.go @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package generate + +import ( + "fmt" + "os" + + devtools "github.com/elastic/beats/dev-tools/mage" + genfileset "github.com/elastic/beats/filebeat/generator/fileset" +) + +// Fileset creates a new fileset for an existing Filebeat module. +// Use MODULE=module to specify the name of the existing module +// Use FILESET=fileset to specify the name of the new fileset +func Fileset() error { + targetModule := os.Getenv("MODULE") + targetFileset := os.Getenv("FILESET") + + if targetModule == "" || targetFileset == "" { + return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset createFileset") + } + + ossDir := devtools.OSSBeatDir() + xPackDir := devtools.XPackBeatDir() + + switch devtools.CWD() { + case ossDir: + return genfileset.Generate(targetModule, targetFileset, ossDir, ossDir) + case xPackDir: + return genfileset.Generate(targetModule, targetFileset, xPackDir, ossDir) + default: + return fmt.Errorf("you must be in a filebeat directory") + } +} diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/mage/generate/module.go similarity index 52% rename from filebeat/scripts/generator/fields/main.go rename to filebeat/scripts/mage/generate/module.go index 523ebf82249..5d14e3ec09f 100644 --- a/filebeat/scripts/generator/fields/main.go +++ b/filebeat/scripts/mage/generate/module.go @@ -15,38 +15,33 @@ // specific language governing permissions and limitations // under the License. -package main +package generate import ( - "flag" "fmt" "os" - "github.com/elastic/beats/filebeat/generator/fields" + devtools "github.com/elastic/beats/dev-tools/mage" + genmod "github.com/elastic/beats/filebeat/generator/module" ) -func main() { - module := flag.String("module", "", "Name of the module") - fileset := flag.String("fileset", "", "Name of the fileset") - beatsPath := flag.String("beats_path", ".", "Path to elastic/beats") - noDoc := flag.Bool("nodoc", false, "Generate documentation for fields") - flag.Parse() - - if *module == "" { - fmt.Println("Missing parameter: module") - os.Exit(1) +// Module creates a new Filebeat module. +// Use MODULE=module to specify the name of the new module +func Module() error { + targetModule := os.Getenv("MODULE") + if targetModule == "" { + return fmt.Errorf("you must specify the module: MODULE=name mage generate:module") } - if *fileset == "" { - fmt.Println("Missing parameter: fileset") - os.Exit(1) - } + ossDir := devtools.OSSBeatDir() + xPackDir := devtools.XPackBeatDir() - err := fields.Generate(*beatsPath, *module, *fileset, *noDoc) - if err != nil { - fmt.Printf("Error while generating fields.yml: %v\n", err) - os.Exit(2) + switch devtools.CWD() { + case ossDir: + return genmod.Generate(targetModule, ossDir, ossDir) + case xPackDir: + return genmod.Generate(targetModule, xPackDir, ossDir) + default: + return fmt.Errorf("you must be in a filebeat directory") } - - fmt.Printf("Fields.yml generated for %s/%s\n", *module, *fileset) } diff --git a/go.mod b/go.mod index 22fdc02dfee..288e93230aa 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/elastic/go-structform v0.0.6 github.com/elastic/go-sysinfo v1.3.0 github.com/elastic/go-txfile v0.0.7 - github.com/elastic/go-ucfg v0.8.1 + github.com/elastic/go-ucfg v0.8.2 github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.10.5 github.com/fatih/color v1.5.0 diff --git a/go.sum b/go.sum index 2309e6a135c..8b40f5c2615 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/elastic/go-txfile v0.0.7 h1:Yn28gclW7X0Qy09nSMSsx0uOAvAGMsp6XHydbiLVe github.com/elastic/go-txfile v0.0.7/go.mod h1:H0nCoFae0a4ga57apgxFsgmRjevNCsEaT6g56JoeKAE= github.com/elastic/go-ucfg v0.7.0 h1:1+C/sZdJKww8hKl7XtLPTjs4cFslhQF2fazKTF+ZE+4= github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/go-ucfg v0.8.1 h1:qcu7ESw8PKSqVrccHhSmTmkZS3Wa2ip+6eJ3FXcbrUI= -github.com/elastic/go-ucfg v0.8.1/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= +github.com/elastic/go-ucfg v0.8.2 h1:zVg4HdUfwQ0/SHKulInChTRLDh+tNZ1AAbj1fNWwPyQ= +github.com/elastic/go-ucfg v0.8.2/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index af10819b81a..60941544931 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -655,6 +655,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -748,6 +754,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 9270a862925..949df01a29a 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -449,6 +449,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -542,6 +548,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/journalbeat/reader/journal.go b/journalbeat/reader/journal.go index cd94154efd1..c7a34c8bfb2 100644 --- a/journalbeat/reader/journal.go +++ b/journalbeat/reader/journal.go @@ -288,8 +288,16 @@ func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{} if fc.isInteger { v, err := strconv.ParseInt(value, 10, 64) if err != nil { - r.logger.Debugf("Failed to convert field: %s \"%v\" to int: %v", fc.name, value, err) - return value + // On some versions of systemd the 'syslog.pid' can contain the username + // appended to the end of the pid. In most cases this does not occur + // but in the cases that it does, this tries to strip ',\w*' from the + // value and then perform the conversion. + s := strings.Split(value, ",") + v, err = strconv.ParseInt(s[0], 10, 64) + if err != nil { + r.logger.Debugf("Failed to convert field: %s \"%v\" to int: %v", fc.name, value, err) + return value + } } return v } diff --git a/journalbeat/reader/journal_test.go b/journalbeat/reader/journal_test.go index 85846501957..d7619c45829 100644 --- a/journalbeat/reader/journal_test.go +++ b/journalbeat/reader/journal_test.go @@ -57,6 +57,45 @@ func TestToEvent(t *testing.T) { }, }, }, + // 'syslog.pid' field without user append + ToEventTestCase{ + entry: sdjournal.JournalEntry{ + Fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456", + }, + }, + expectedFields: common.MapStr{ + "syslog": common.MapStr{ + "pid": int64(123456), + }, + }, + }, + // 'syslog.pid' field with user append + ToEventTestCase{ + entry: sdjournal.JournalEntry{ + Fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456,root", + }, + }, + expectedFields: common.MapStr{ + "syslog": common.MapStr{ + "pid": int64(123456), + }, + }, + }, + // 'syslog.pid' field empty + ToEventTestCase{ + entry: sdjournal.JournalEntry{ + Fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "", + }, + }, + expectedFields: common.MapStr{ + "syslog": common.MapStr{ + "pid": "", + }, + }, + }, // custom field ToEventTestCase{ entry: sdjournal.JournalEntry{ diff --git a/libbeat/_meta/config.reference.yml.tmpl b/libbeat/_meta/config.reference.yml.tmpl index ede5934fd24..a949cd43327 100644 --- a/libbeat/_meta/config.reference.yml.tmpl +++ b/libbeat/_meta/config.reference.yml.tmpl @@ -391,6 +391,12 @@ output.elasticsearch: # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 {{if not .ExcludeLogstash}} #----------------------------- Logstash output --------------------------------- #output.logstash: @@ -485,6 +491,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/libbeat/common/transport/tlscommon/ca_pinning.go b/libbeat/common/transport/tlscommon/ca_pinning.go new file mode 100644 index 00000000000..d83bf533d13 --- /dev/null +++ b/libbeat/common/transport/tlscommon/ca_pinning.go @@ -0,0 +1,73 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/base64" + + "github.com/pkg/errors" +) + +// ErrCAPinMissmatch is returned when no pin is matched in the verified chain. +var ErrCAPinMissmatch = errors.New("provided CA certificate pins doesn't match any of the certificate authorities used to validate the certificate") + +type pins []string + +func (p pins) Matches(candidate string) bool { + for _, pin := range p { + if pin == candidate { + return true + } + } + return false +} + +// verifyPeerCertFunc is a callback defined on the tls.Config struct that will called when a +// TLS connection is used. +type verifyPeerCertFunc func([][]byte, [][]*x509.Certificate) error + +// MakeCAPinCallback loops through the verified chains and will try to match the certificates pin. +// +// NOTE: Defining a PIN to check certificates is not a replacement for the normal TLS validations it's +// an additional validation. In fact if you set `InsecureSkipVerify` to true and a PIN, the +// verifiedChains variable will be empty and the added validation will fail. +func MakeCAPinCallback(hashes pins) func([][]byte, [][]*x509.Certificate) error { + return func(_ [][]byte, verifiedChains [][]*x509.Certificate) error { + // The chain of trust has been already established before the call to the VerifyPeerCertificate + // function, after we go through the chain to make sure we have at least a certificate certificate + // that match the provided pin. + for _, chain := range verifiedChains { + for _, certificate := range chain { + h := Fingerprint(certificate) + if hashes.Matches(h) { + return nil + } + } + } + + return ErrCAPinMissmatch + } +} + +// Fingerprint takes a certificate and create a hash of the DER encoded public key. +func Fingerprint(certificate *x509.Certificate) string { + hash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) + return base64.StdEncoding.EncodeToString(hash[:]) +} diff --git a/libbeat/common/transport/tlscommon/ca_pinning_test.go b/libbeat/common/transport/tlscommon/ca_pinning_test.go new file mode 100644 index 00000000000..a9f95e5c6ac --- /dev/null +++ b/libbeat/common/transport/tlscommon/ca_pinning_test.go @@ -0,0 +1,396 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "io/ioutil" + "math/big" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" +) + +var ser int64 = 1 + +func TestCAPinning(t *testing.T) { + host := "127.0.0.1" + + t.Run("when the ca_sha256 field is not defined we use normal certificate validation", + func(t *testing.T) { + cfg := common.MustNewConfigFrom(map[string]interface{}{ + "certificate_authorities": []string{"ca_test.pem"}, + }) + + config := &Config{} + err := cfg.Unpack(config) + require.NoError(t, err) + + tlsCfg, err := LoadTLSConfig(config) + require.NoError(t, err) + + tls := tlsCfg.BuildModuleConfig(host) + require.Nil(t, tls.VerifyPeerCertificate) + }) + + t.Run("when the ca_sha256 field is defined we use CA cert pinning", func(t *testing.T) { + cfg := common.MustNewConfigFrom(map[string]interface{}{ + "ca_sha256": "hello", + }) + + config := &Config{} + err := cfg.Unpack(config) + require.NoError(t, err) + + tlsCfg, err := LoadTLSConfig(config) + require.NoError(t, err) + + tls := tlsCfg.BuildModuleConfig(host) + require.NotNil(t, tls.VerifyPeerCertificate) + }) + + t.Run("CA Root -> Certificate and we have the CA root pin", func(t *testing.T) { + msg := []byte("OK received message") + + ca, err := genCA() + require.NoError(t, err) + + serverCert, err := genSignedCert(ca, x509.KeyUsageDigitalSignature, false) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(msg) + }) + + // Select a random available port from the OS. + addr := "localhost:0" + + l, err := net.Listen("tcp", addr) + + server := &http.Server{ + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{ + serverCert, + }, + }, + } + + // Start server and shut it down when the tests are over. + go server.ServeTLS(l, "", "") + defer l.Close() + + // Root CA Pool + require.NoError(t, err) + rootCAs := x509.NewCertPool() + rootCAs.AddCert(ca.Leaf) + + // Get the pin of the RootCA. + pin := Fingerprint(ca.Leaf) + + tlsC := &TLSConfig{ + RootCAs: rootCAs, + CASha256: []string{pin}, + } + + config := tlsC.BuildModuleConfig("localhost") + hostToConnect := l.Addr().String() + + transport := &http.Transport{ + TLSClientConfig: config, + } + + client := &http.Client{Transport: transport} + + port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") + + req, err := http.NewRequest("GET", "https://localhost:"+port, nil) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + content, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + assert.True(t, bytes.Equal(msg, content)) + + // 1. create key-pair + // 2. create pin + // 3. start server + // 4. Connect + // 5. Check wrong key do not work + // 6. Check good key work + // 7. check plain text fails to work. + }) + + t.Run("CA Root -> Intermediate -> Certificate and we receive the CA Root Pin", func(t *testing.T) { + msg := []byte("OK received message") + + ca, err := genCA() + require.NoError(t, err) + + intermediate, err := genSignedCert(ca, x509.KeyUsageDigitalSignature|x509.KeyUsageCertSign, true) + require.NoError(t, err) + + serverCert, err := genSignedCert(intermediate, x509.KeyUsageDigitalSignature, false) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(msg) + }) + + // Select a random available port from the OS. + addr := "localhost:0" + + l, err := net.Listen("tcp", addr) + require.NoError(t, err) + + // Server needs to provides the chain of trust, so server certificate + intermediate. + // RootCAs will trust the intermediate, intermediate will trust the server. + serverCert.Certificate = append(serverCert.Certificate, intermediate.Certificate...) + + server := &http.Server{ + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{ + serverCert, + }, + }, + } + + // Start server and shut it down when the tests are over. + go server.ServeTLS(l, "", "") + defer l.Close() + + // Root CA Pool + rootCAs := x509.NewCertPool() + rootCAs.AddCert(ca.Leaf) + + // Get the pin of the RootCA. + pin := Fingerprint(ca.Leaf) + + tlsC := &TLSConfig{ + RootCAs: rootCAs, + CASha256: []string{pin}, + } + + config := tlsC.BuildModuleConfig("localhost") + hostToConnect := l.Addr().String() + + transport := &http.Transport{ + TLSClientConfig: config, + } + + client := &http.Client{Transport: transport} + + port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") + + req, err := http.NewRequest("GET", "https://localhost:"+port, nil) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + content, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + assert.True(t, bytes.Equal(msg, content)) + }) + + t.Run("When we have the wrong pin we refuse to connect", func(t *testing.T) { + msg := []byte("OK received message") + + ca, err := genCA() + require.NoError(t, err) + + intermediate, err := genSignedCert(ca, x509.KeyUsageDigitalSignature|x509.KeyUsageCertSign, true) + require.NoError(t, err) + + serverCert, err := genSignedCert(intermediate, x509.KeyUsageDigitalSignature, false) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(msg) + }) + + // Select a random available port from the OS. + addr := "localhost:0" + + l, err := net.Listen("tcp", addr) + require.NoError(t, err) + + // Server needs to provides the chain of trust, so server certificate + intermediate. + // RootCAs will trust the intermediate, intermediate will trust the server. + serverCert.Certificate = append(serverCert.Certificate, intermediate.Certificate...) + + server := &http.Server{ + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{ + serverCert, + }, + }, + } + + // Start server and shut it down when the tests are over. + go server.ServeTLS(l, "", "") + defer l.Close() + + // Root CA Pool + rootCAs := x509.NewCertPool() + rootCAs.AddCert(ca.Leaf) + + // Get the pin of the RootCA. + pin := "wrong-pin" + + tlsC := &TLSConfig{ + RootCAs: rootCAs, + CASha256: []string{pin}, + } + + config := tlsC.BuildModuleConfig("localhost") + hostToConnect := l.Addr().String() + + transport := &http.Transport{ + TLSClientConfig: config, + } + + client := &http.Client{Transport: transport} + + port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") + + req, err := http.NewRequest("GET", "https://localhost:"+port, nil) + require.NoError(t, err) + _, err = client.Do(req) + require.Error(t, err) + }) +} + +func genCA() (tls.Certificate, error) { + ca := &x509.Certificate{ + SerialNumber: serial(), + Subject: pkix.Name{ + CommonName: "localhost", + Organization: []string{"TESTING"}, + Country: []string{"CANADA"}, + Province: []string{"QUEBEC"}, + Locality: []string{"MONTREAL"}, + StreetAddress: []string{"testing road"}, + PostalCode: []string{"HOH OHO"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(1 * time.Hour), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) // less secure key for quicker testing. + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "fail to generate RSA key") + } + + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "fail to create certificate") + } + + leaf, err := x509.ParseCertificate(caBytes) + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "fail to parse certificate") + } + + return tls.Certificate{ + Certificate: [][]byte{caBytes}, + PrivateKey: caKey, + Leaf: leaf, + }, nil +} + +// genSignedCert generates a CA and KeyPair and remove the need to depends on code of agent. +func genSignedCert(ca tls.Certificate, keyUsage x509.KeyUsage, isCA bool) (tls.Certificate, error) { + // Create another Cert/key + cert := &x509.Certificate{ + SerialNumber: big.NewInt(2000), + Subject: pkix.Name{ + CommonName: "localhost", + Organization: []string{"TESTING"}, + Country: []string{"CANADA"}, + Province: []string{"QUEBEC"}, + Locality: []string{"MONTREAL"}, + StreetAddress: []string{"testing road"}, + PostalCode: []string{"HOH OHO"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(1 * time.Hour), + IsCA: isCA, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: keyUsage, + BasicConstraintsValid: true, + } + + certKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "fail to generate RSA key") + } + + certBytes, err := x509.CreateCertificate( + rand.Reader, + cert, + ca.Leaf, + &certKey.PublicKey, + ca.PrivateKey, + ) + + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "fail to create signed certificate") + } + + leaf, err := x509.ParseCertificate(certBytes) + if err != nil { + return tls.Certificate{}, errors.Wrap(err, "fail to parse the certificate") + } + + return tls.Certificate{ + Certificate: [][]byte{certBytes}, + PrivateKey: certKey, + Leaf: leaf, + }, nil +} + +func serial() *big.Int { + ser = ser + 1 + return big.NewInt(ser) +} diff --git a/libbeat/common/transport/tlscommon/config.go b/libbeat/common/transport/tlscommon/config.go index 38f6500e183..3fdaeced560 100644 --- a/libbeat/common/transport/tlscommon/config.go +++ b/libbeat/common/transport/tlscommon/config.go @@ -33,6 +33,7 @@ type Config struct { Certificate CertificateConfig `config:",inline" yaml:",inline"` CurveTypes []tlsCurveType `config:"curve_types" yaml:"curve_types,omitempty"` Renegotiation tlsRenegotiationSupport `config:"renegotiation" yaml:"renegotiation"` + CASha256 pins `config:"ca_sha256" yaml:"ca_sha256,omitempty"` } // LoadTLSConfig will load a certificate from config with all TLS based keys @@ -88,6 +89,7 @@ func LoadTLSConfig(config *Config) (*TLSConfig, error) { CipherSuites: cipherSuites, CurvePreferences: curves, Renegotiation: tls.RenegotiationSupport(config.Renegotiation), + CASha256: config.CASha256, }, nil } diff --git a/libbeat/common/transport/tlscommon/tls_config.go b/libbeat/common/transport/tlscommon/tls_config.go index a7e69ea3ab4..41c574bc078 100644 --- a/libbeat/common/transport/tlscommon/tls_config.go +++ b/libbeat/common/transport/tlscommon/tls_config.go @@ -64,6 +64,10 @@ type TLSConfig struct { // ClientAuth controls how we want to verify certificate from a client, `none`, `optional` and // `required`, default to required. Do not affect TCP client. ClientAuth tls.ClientAuthType + + // CASha256 is the CA certificate pin, this is used to validate the CA that will be used to trust + // the server certificate. + CASha256 pins } // ToConfig generates a tls.Config object. Note, you must use BuildModuleConfig to generate a config with @@ -79,17 +83,25 @@ func (c *TLSConfig) ToConfig() *tls.Config { logp.Warn("SSL/TLS verifications disabled.") } + // When we are usign the CAsha256 pin to validate the CA used to validate the chain + // we add a custom callback. + var verifyPeerCertFn verifyPeerCertFunc + if len(c.CASha256) > 0 { + verifyPeerCertFn = MakeCAPinCallback(c.CASha256) + } + return &tls.Config{ - MinVersion: minVersion, - MaxVersion: maxVersion, - Certificates: c.Certificates, - RootCAs: c.RootCAs, - ClientCAs: c.ClientCAs, - InsecureSkipVerify: insecure, - CipherSuites: c.CipherSuites, - CurvePreferences: c.CurvePreferences, - Renegotiation: c.Renegotiation, - ClientAuth: c.ClientAuth, + MinVersion: minVersion, + MaxVersion: maxVersion, + Certificates: c.Certificates, + RootCAs: c.RootCAs, + ClientCAs: c.ClientCAs, + InsecureSkipVerify: insecure, + CipherSuites: c.CipherSuites, + CurvePreferences: c.CurvePreferences, + Renegotiation: c.Renegotiation, + ClientAuth: c.ClientAuth, + VerifyPeerCertificate: verifyPeerCertFn, } } diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml index e0200a1b89b..267cc441a15 100644 --- a/libbeat/docker-compose.yml +++ b/libbeat/docker-compose.yml @@ -20,6 +20,8 @@ services: - KIBANA_PORT=5601 - ES_MONITORING_HOST=elasticsearch_monitoring - ES_MONITORING_PORT=9200 + - ES_HOST_SSL=elasticsearchssl + - ES_PORT_SSL=9200 env_file: - ${PWD}/build/test.env volumes: @@ -36,6 +38,7 @@ services: depends_on: elasticsearch: { condition: service_healthy } elasticsearch_monitoring: { condition: service_healthy } + elasticsearchssl: { condition: service_healthy } logstash: { condition: service_healthy } kafka: { condition: service_healthy } redis: { condition: service_healthy } @@ -57,6 +60,36 @@ services: healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] + elasticsearchssl: + extends: + file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml + service: elasticsearch + healthcheck: + test: ["CMD", "curl", "-u", "admin:changeme", "-f", "https://localhost:9200", "--insecure"] + retries: 1200 + interval: 5s + start_period: 60s + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=" + - "transport.host=127.0.0.1" + - "http.host=0.0.0.0" + - "xpack.security.enabled=true" + - "indices.id_field_data.enabled=true" + - "xpack.license.self_generated.type=trial" + - "xpack.security.http.ssl.enabled=true" + - "xpack.security.http.ssl.key=/usr/share/elasticsearch/config/pki/elasticsearchssl/elasticsearchssl.key" + - "xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/pki/elasticsearchssl/elasticsearchssl.crt" + - "xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/pki/ca/ca.crt" + # Do no used indices as the storage for credentials, using file based allow Elasticsearch + # to be online and green much quicker. + - "xpack.security.authc.realms.file.file1.order=0" + volumes: + - ${ES_BEATS}/testing/environments/docker/elasticsearch/pki:/usr/share/elasticsearch/config/pki:ro + ports: + - 9200 + command: bash -c "bin/elasticsearch-users useradd admin -r superuser -p changeme | /usr/local/bin/docker-entrypoint.sh eswrapper" + # This host name is static because of the certificate. logstash: extends: diff --git a/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc b/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc index 7d72610fbd7..0f79347e3a1 100644 --- a/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc +++ b/libbeat/docs/release-notes/breaking/breaking-7.6.asciidoc @@ -12,4 +12,18 @@ //tag::notable-breaking-changes[] +[float] +==== Version removed from default ILM policy name + +To prevent custom ILM policies from breaking during upgrades, the default +ILM policy name no longer includes the {beats} version. + +If you want to include the version, use the `setup.ilm.policy_name` +configuration setting. For example: + +[source,yaml] +---- +setup.ilm.policy_name: "%{[agent.name]}-%{[agent.version]}" +---- + // end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/highlights/highlights-7.6.0.asciidoc b/libbeat/docs/release-notes/highlights/highlights-7.6.0.asciidoc index 2890861b3f2..e209c82a06c 100644 --- a/libbeat/docs/release-notes/highlights/highlights-7.6.0.asciidoc +++ b/libbeat/docs/release-notes/highlights/highlights-7.6.0.asciidoc @@ -7,8 +7,8 @@ Each release of {beats} brings new features and product improvements. Following are the most notable features and enhancements in 7.6. -For a complete list of highlights, see the -https://www.elastic.co/blog/beats-7-6-0-released[{beats} 7.6 release blog]. +For a complete list of related highlights, see the +https://www.elastic.co/blog/elastic-observability-7-6-0-released[Observability 7.6 release blog]. For a list of bug fixes and other changes, see the {beats} <> and <>. @@ -18,8 +18,67 @@ For a list of bug fixes and other changes, see the {beats} // tag::notable-highlights[] -//[float] -//==== Highlight +[float] +==== Support added for TLS 1.3 + +In 7.6, we've added support for TLS 1.3, enabling you to take advantage +of the security and performance improvements available in TLS 1.3 when {beats} +connects to an {es} cluster that supports it. + +TLS 1.1, 1.2, and 1.3 are now enabled by default. + +[float] +==== Azure cloud monitoring improvements + +The new {metricbeat-ref}/metricbeat-metricset-azure-storage.html[`storage`] +metricset in the {metricbeat} {metricbeat-ref}/metricbeat-module-azure.html[Azure +module] adds the ability to collect metrics from storage accounts. + +We've also made it easier for you to do a cost analysis on metrics +collection by adding a cost warning message to each metrics API call. The +message indicates the cost applied while retrieving metric values from Azure +resources. + +[float] +==== Google cloud monitoring improvements + +Starting with 7.6, you can deploy {functionbeat} as a Google Function via Google +Cloud Deployment Manager and pull log events from +https://cloud.google.com/pubsub/[Google Pub/Sub] and +https://cloud.google.com/storage/[Google Cloud Storage]. + +We've also expanded support in {metricbeat} by adding a beta release of the +{metricbeat-ref}/metricbeat-module-googlecloud.html[Google Cloud Platform module]. +This module fetches monitoring metrics from Google Cloud Platform (GCP) by using +the +https://cloud.google.com/monitoring/api/metrics_gcp[Stackdriver Monitoring API]. + +On the logging side, we've added two new filesets to the {filebeat} +{filebeat-ref}/filebeat-module-googlecloud.html[Google Cloud module] to make it +easier for you to ingest Gloogle Cloud logs: + +* The `audit` fileset parses +https://cloud.google.com/logging/docs/audit/[Google Cloud Audit Logs]. +* The `firewall` fileset parses firewall logs generated by firewall rules +logging. + +[float] +==== Better support for document deduplication + +To help prevent duplicate events, we've introduced `document_id` settings +in {beats} that you can use to set the document ID _before_ sending events to +an output. The ID is stored in the {beats} `@metadata._id` field and used to +set the document ID during indexing. Both the `decode_json_fields` processor and +`json` options in the {filebeat} have been enhanced to include a +`document_id` setting to use when decoding JSON data. + +We've also added new processors for generating IDs when your data has no natural +key field. The {filebeat-ref}/add-id.html[`add_id`] processor generates a unique +ID for an event. The {filebeat-ref}/fingerprint.html[`fingerprint`] processor +generates a fingerprint of an event based on a specified subset of its fields. + +To learn more, see {filebeat-ref}/filebeat-deduplication.html[Data +deduplication]. //Description diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index 9e7087e72be..5ddda476ddd 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -362,6 +362,51 @@ ifdef::autodiscoverHints[] include::../../{beatname_lc}/docs/autodiscover-hints.asciidoc[] endif::autodiscoverHints[] +ifdef::autodiscoverAWSEC2[] +[float] +===== Amazon EC2s + +*Note: This provider is experimental* + +The Amazon EC2 autodiscover provider discovers https://aws.amazon.com/ec2/[EC2 instances]. +This is useful for users to launch Metricbeat modules to monitor services running on AWS EC2 instances. +For example, to gather MySQL metrics from mysql servers running on EC2 instances with specific tag `service: mysql`. + +This provider will load AWS credentials using the standard AWS environment variables and shared credentials files +see https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html[Best Practices for Managing AWS Access Keys] +for more information. If you do not wish to use these, you may explicitly set the `access_key_id` and +`secret_access_key` variables. + +These are the available fields during within config templating. +The `aws.ec2.*` fields and `cloud.*` fields will be available on each emitted event. + +* cloud.availability_zone +* cloud.instance.id +* cloud.machine.type +* cloud.provider +* cloud.region + +* aws.ec2.architecture +* aws.ec2.image.id +* aws.ec2.kernel.id +* aws.ec2.monitoring.state +* aws.ec2.private.dns_name +* aws.ec2.private.ip +* aws.ec2.public.dns_name +* aws.ec2.public.ip +* aws.ec2.root_device_name +* aws.ec2.state.code +* aws.ec2.state.name +* aws.ec2.subnet.id +* aws.ec2.tags +* aws.ec2.vpc.id + +include::../../{beatname_lc}/docs/autodiscover-aws-ec2-config.asciidoc[] + +This autodiscover provider takes our standard <>. + +endif::autodiscoverAWSEC2[] + [[configuration-autodiscover-advanced]] === Advanced usage diff --git a/libbeat/docs/shared-ssl-config.asciidoc b/libbeat/docs/shared-ssl-config.asciidoc index e0ac6b85c49..899d0dcd681 100644 --- a/libbeat/docs/shared-ssl-config.asciidoc +++ b/libbeat/docs/shared-ssl-config.asciidoc @@ -229,6 +229,20 @@ are `never`, `once`, and `freely`. The default value is never. * `once` - Allows a remote server to request renegotiation once per connection. * `freely` - Allows a remote server to repeatedly request renegotiation. + +[float] +==== `ca_sha256` + +This configure a certificate pin can that ca be used to ensure that a specific certificate is used +to as part of the verified chain. + +The pin is a base64 encoded string of the SHA-256 of the certificate. + +NOTE: This check is not a replacement for the normal SSL validation but it add additional validation. +If this option is used with `verification_mode` set to `none`, the check will always fail because +it will not receive any verified chains. + + ifeval::["{beatname_lc}" == "filebeat"] [float] ==== `client_authentication` diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 69ddfe6745e..d52a872efc8 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -11,7 +11,7 @@ connecting to Elasticsearch. ifndef::no-output-logstash[] NOTE: A connection to Elasticsearch is required to load the index template. If -the output is Logstash, you must <>. endif::[] diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 1191b1ae011..ad1ff3e45de 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -79,7 +79,7 @@ func newKafkaClient( hosts: hosts, topic: topic, key: key, - index: index, + index: strings.ToLower(index), codec: writer, config: *cfg, } diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index 598413f5814..6d2a30d39ea 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -18,6 +18,7 @@ package logstash import ( + "strings" "time" "github.com/elastic/beats/libbeat/beat" @@ -80,7 +81,7 @@ func readConfig(cfg *common.Config, info beat.Info) (*Config, error) { } if c.Index == "" { - c.Index = info.IndexPrefix + c.Index = strings.ToLower(info.IndexPrefix) } return &c, nil diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index b9fa409b6e9..fb42626ec5c 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -18,6 +18,8 @@ package logstash import ( + "strings" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs/codec/json" ) @@ -27,6 +29,7 @@ func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) fun Pretty: false, EscapeHTML: escapeHTML, }) + index = strings.ToLower(index) return func(event interface{}) (d []byte, err error) { d, err = enc.Encode(index, event.(*beat.Event)) if err != nil { diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 8941984b922..6aca15337fe 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -92,7 +92,8 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexSel := outil.MakeSelector(outil.FmtSelectorExpr(indexFmt, "")) + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, }) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index d6ff4931b64..d06ee4e3209 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,6 +19,7 @@ package outil import ( "fmt" + "strings" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -26,10 +27,14 @@ import ( "github.com/elastic/beats/libbeat/conditions" ) +// Selector is used to produce a string based on the contents of a Beats event. +// A selector supports multiple rules that need to be configured. type Selector struct { sel SelectorExpr } +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. type Settings struct { // single selector key and default option keyword Key string @@ -44,6 +49,8 @@ type Settings struct { FailEmpty bool } +// SelectorExpr represents an expression object that can be composed with other +// expressions in order to build a Selector. type SelectorExpr interface { sel(evt *beat.Event) (string, error) } @@ -76,6 +83,7 @@ type mapSelector struct { var nilSelector SelectorExpr = &emptySelector{} +// MakeSelector creates a selector from a set of selector expressions. func MakeSelector(es ...SelectorExpr) Selector { switch len(es) { case 0: @@ -95,10 +103,12 @@ func (s Selector) Select(evt *beat.Event) (string, error) { return s.sel.sel(evt) } +// IsEmpty checks if the selector is not configured and will always return an empty string. func (s Selector) IsEmpty() bool { return s.sel == nilSelector || s.sel == nil } +// IsConst checks if the selector will always return the same string. func (s Selector) IsConst() bool { if s.sel == nilSelector { return true @@ -108,6 +118,7 @@ func (s Selector) IsConst() bool { return ok } +// BuildSelectorFromConfig creates a selector from a configuration object. func BuildSelectorFromConfig( cfg *common.Config, settings Settings, @@ -156,17 +167,13 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - if fmtstr.IsConst() { - str, err := fmtstr.Run(nil) - if err != nil { - return Selector{}, err - } + fmtsel, err := FmtSelectorExpr(fmtstr, "") + if err != nil { + return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } - if str != "" { - sel = append(sel, ConstSelectorExpr(str)) - } - } else { - sel = append(sel, FmtSelectorExpr(fmtstr, "")) + if fmtsel != nilSelector { + sel = append(sel, fmtsel) } } @@ -183,22 +190,44 @@ func BuildSelectorFromConfig( return MakeSelector(sel...), nil } +// EmptySelectorExpr create a selector expression that returns an empty string. func EmptySelectorExpr() SelectorExpr { return nilSelector } +// ConstSelectorExpr creates a selector expression that always returns the configured string. func ConstSelectorExpr(s string) SelectorExpr { - return &constSelector{s} + if s == "" { + return EmptySelectorExpr() + } + return &constSelector{strings.ToLower(s)} } -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) SelectorExpr { - return &fmtSelector{*fmt, fallback} +// FmtSelectorExpr creates a selector expression using a format string. If the +// event can not be applied the default fallback constant string will be returned. +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { + if fmt.IsConst() { + str, err := fmt.Run(nil) + if err != nil { + return nil, err + } + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil } +// ConcatSelectorExpr combines multiple expressions that are run one after the other. +// The first expression that returns a string wins. func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { return &listSelector{s} } +// ConditionalSelectorExpr executes the given expression only if the event +// matches the given condition. func ConditionalSelectorExpr( s SelectorExpr, cond conditions.Condition, @@ -206,12 +235,39 @@ func ConditionalSelectorExpr( return &condSelector{s, cond} } +// LookupSelectorExpr replaces the produced string with an table entry. +// If there is no entry in the table the default fallback string will be reported. func LookupSelectorExpr( - s SelectorExpr, + evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) SelectorExpr { - return &mapSelector{s, fallback, table} +) (SelectorExpr, error) { + if evtfmt.IsConst() { + str, err := evtfmt.Run(nil) + if err != nil { + return nil, err + } + + str = table[strings.ToLower(str)] + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &mapSelector{ + from: &fmtSelector{f: *evtfmt}, + to: table, + otherwise: fallback, + }, nil +} + +func lowercaseTable(table map[string]string) map[string]string { + tmp := make(map[string]string, len(table)) + for k, v := range table { + tmp[strings.ToLower(k)] = strings.ToLower(v) + } + return tmp } func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { @@ -239,7 +295,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = tmp + otherwise = strings.ToLower(tmp) } // 3. extract optional `mapping` @@ -276,45 +332,14 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - str = mapping.Table[str] - if str == "" { - str = otherwise - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = &mapSelector{ - from: FmtSelectorExpr(evtfmt, ""), - to: mapping.Table, - otherwise: otherwise, - } - } + sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) } else { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = FmtSelectorExpr(evtfmt, otherwise) - } + sel, err = FmtSelectorExpr(evtfmt, otherwise) } + if err != nil { + return nil, err + } + if cond != nil && sel != nilSelector { sel = ConditionalSelectorExpr(sel, cond) } @@ -363,7 +388,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return n, nil + return strings.ToLower(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index d4093ed5660..f6e837966b2 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,74 +31,92 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string event common.MapStr expected string }{ - { - "constant key", + "constant key": { `key: value`, common.MapStr{}, "value", }, - { - "format string key", + "lowercase constant key": { + `key: VaLuE`, + common.MapStr{}, + "value", + }, + "format string key": { `key: '%{[key]}'`, common.MapStr{"key": "value"}, "value", }, - { - "key with empty keys", + "lowercase format string key": { + `key: '%{[key]}'`, + common.MapStr{"key": "VaLuE"}, + "value", + }, + "key with empty keys": { `{key: value, keys: }`, common.MapStr{}, "value", }, - { - "constant in multi key", + "lowercase key with empty keys": { + `{key: vAlUe, keys: }`, + common.MapStr{}, + "value", + }, + "constant in multi key": { `keys: [key: 'value']`, common.MapStr{}, "value", }, - { - "format string in multi key", + "format string in multi key": { `keys: [key: '%{[key]}']`, common.MapStr{"key": "value"}, "value", }, - { - "missing format string key with default in rule", + "missing format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{}, "value", }, - { - "empty format string key with default in rule", + "lowercase missing format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAlUe`, + common.MapStr{}, + "value", + }, + "empty format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{"key": ""}, "value", }, - { - "missing format string key with constant in next rule", + "lowercase empty format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAluE`, + common.MapStr{"key": ""}, + "value", + }, + "missing format string key with constant in next rule": { `keys: - key: '%{[key]}' - key: value`, common.MapStr{}, "value", }, - { - "missing format string key with constant in top-level rule", + "missing format string key with constant in top-level rule": { `{ key: value, keys: [key: '%{[key]}']}`, common.MapStr{}, "value", }, - { - "apply mapping", + "apply mapping": { `keys: - key: '%{[key]}' mappings: @@ -106,8 +124,15 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping with default on empty key", + "lowercase applied mapping": { + `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + common.MapStr{"key": "v"}, + "value", + }, + "apply mapping with default on empty key": { `keys: - key: '%{[key]}' default: value @@ -116,8 +141,16 @@ func TestSelector(t *testing.T) { common.MapStr{"key": ""}, "value", }, - { - "apply mapping with default on empty lookup", + "lowercase apply mapping with default on empty key": { + `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + common.MapStr{"key": ""}, + "value", + }, + "apply mapping with default on empty lookup": { `keys: - key: '%{[key]}' default: value @@ -126,8 +159,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping without match", + "apply mapping without match": { `keys: - key: '%{[key]}' mappings: @@ -136,8 +168,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "x"}, "value", }, - { - "mapping with constant key", + "mapping with constant key": { `keys: - key: k mappings: @@ -145,8 +176,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key", + "mapping with missing constant key": { `keys: - key: unknown mappings: {k: wrong} @@ -154,8 +184,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key, but default", + "mapping with missing constant key, but default": { `keys: - key: unknown default: value @@ -163,16 +192,14 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "matching condition", + "matching condition": { `keys: - key: value when.equals.test: test`, common.MapStr{"test": "test"}, "value", }, - { - "failing condition", + "failing condition": { `keys: - key: wrong when.equals.test: test @@ -182,113 +209,98 @@ func TestSelector(t *testing.T) { }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) - - yaml := strings.Replace(test.config, "\t", " ", -1) - cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") - if err != nil { - t.Errorf("YAML parse error: %v\n%v", err, yaml) - continue - } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + yaml := strings.Replace(test.config, "\t", " ", -1) + cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") + if err != nil { + t.Fatalf("YAML parse error: %v\n%v", err, yaml) + } - sel, err := BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) - if err != nil { - t.Error(err) - continue - } + sel, err := BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + if err != nil { + t.Fatal(err) + } - event := beat.Event{ - Timestamp: time.Now(), - Fields: test.event, - } - actual, err := sel.Select(&event) - if err != nil { - t.Error(err) - continue - } + event := beat.Event{ + Timestamp: time.Now(), + Fields: test.event, + } + actual, err := sel.Select(&event) + if err != nil { + t.Fatal(err) + } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.expected, actual) + }) } } func TestSelectorInitFail(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string }{ - { - "keys missing", + "keys missing": { `test: no key`, }, - { - "invalid keys type", + "invalid keys type": { `keys: 5`, }, - { - "invaid keys element type", + "invaid keys element type": { `keys: [5]`, }, - { - "invalid key type", + "invalid key type": { `key: {}`, }, - { - "missing key in list", + "missing key in list": { `keys: [default: value]`, }, - { - "invalid key type in list", + "invalid key type in list": { `keys: [key: {}]`, }, - { - "fail on invalid format string", + "fail on invalid format string": { `key: '%{[abc}'`, }, - { - "fail on invalid format string in list", + "fail on invalid format string in list": { `keys: [key: '%{[abc}']`, }, - { - "default value type mismatch", + "default value type mismatch": { `keys: [{key: ok, default: {}}]`, }, - { - "mappings type mismatch", + "mappings type mismatch": { `keys: - key: '%{[k]}' mappings: {v: {}}`, }, - { - "condition empty", + "condition empty": { `keys: - key: value when:`, }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Fatal(err) + } - cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") - if err != nil { - t.Error(err) - continue - } + _, err = BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) - _, err = BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, + assert.Error(t, err) + t.Log(err) }) - assert.Error(t, err) - t.Log(err) } } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 7238ba367fd..1764d9e60f9 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -21,6 +21,7 @@ import ( "errors" "regexp" "strconv" + "strings" "time" "github.com/garyburd/redigo/redis" @@ -77,7 +78,7 @@ func newClient( observer: observer, timeout: timeout, password: pass, - index: index, + index: strings.ToLower(index), db: db, dataType: dt, key: key, diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index 86712a81fee..078ec38f1cc 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -68,7 +68,7 @@ func init() { processors.RegisterPlugin("decode_json_fields", checks.ConfigChecked(NewDecodeJSONFields, checks.RequireFields("fields"), - checks.AllowedFields("fields", "max_depth", "overwrite_keys", "add_error_key", "process_array", "target", "when"))) + checks.AllowedFields("fields", "max_depth", "overwrite_keys", "add_error_key", "process_array", "target", "when", "document_id"))) jsprocessor.RegisterPlugin("DecodeJSONFields", NewDecodeJSONFields) } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 6201fea6b76..380fb69367d 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -40,6 +40,8 @@ ES_HOST?=elasticsearch ES_PORT?=9200 ES_USER?=beats ES_PASS?=testing +ES_HOST_SSL?=elasticsearchssl +ES_PORT_SSL?=9200 KIBANA_HOST?=kibana KIBANA_PORT?=5601 # Kibana's Elaticsearch user @@ -404,6 +406,8 @@ write-environment: # set ENV variables for beat echo "ES_HOST=${ES_HOST}" >> ${BUILD_DIR}/test.env echo "ES_PORT=${ES_PORT}" >> ${BUILD_DIR}/test.env + echo "ES_HOST_SSL=${ES_HOST_SSL}" >> ${BUILD_DIR}/test.env + echo "ES_PORT_SSL=${ES_PORT_SSL}" >> ${BUILD_DIR}/test.env echo "ES_USER=${ES_USER}" >> ${BUILD_DIR}/test.env echo "ES_PASS=${ES_PASS}" >> ${BUILD_DIR}/test.env echo "ES_SUPERUSER_USER=${ES_SUPERUSER_USER}" >> ${BUILD_DIR}/test.env @@ -423,8 +427,7 @@ env-logs: ${DOCKER_COMPOSE} logs -f fix-permissions: - # Change ownership of all files inside /build folder from root/root to current user/group - docker run -v ${PWD}:/beat alpine:3.4 sh -c "find /beat -user 0 -exec chown -h $(shell id -u):$(shell id -g) {} \;" + sh ${ES_BEATS}/script/fix_permissions.sh "${PWD}" set_version: ## @packaging VERSION=x.y.z set the version of the beat to x.y.z ${ES_BEATS}/dev-tools/set_version ${VERSION} diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index d2f7f5b83fe..4869b3fe9ef 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -644,6 +644,16 @@ def get_elasticsearch_url(self): port=os.getenv("ES_PORT", "9200"), ) + def get_elasticsearch_url_ssl(self): + """ + Returns an elasticsearch.Elasticsearch instance built from the + env variables like the integration tests. + """ + return "https://{host}:{port}".format( + host=os.getenv("ES_HOST_SSL", "localhost"), + port=os.getenv("ES_PORT_SSL", "9205"), + ) + def get_kibana_url(self): """ Returns kibana host URL diff --git a/libbeat/tests/system/test_ca_pinning.py b/libbeat/tests/system/test_ca_pinning.py new file mode 100644 index 00000000000..1e0dd6f6a43 --- /dev/null +++ b/libbeat/tests/system/test_ca_pinning.py @@ -0,0 +1,81 @@ +from base import BaseTest +from idxmgmt import IdxMgmt +import os +from nose.plugins.attrib import attr +import unittest +import logging +from nose.tools import raises +from elasticsearch import RequestError + +INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) + + +class TestCAPinning(BaseTest): + """ + Test beat CA pinning for elasticsearch + """ + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_sending_events_with_a_good_sha256(self): + """ + Test Sending events while using ca pinning with a good sha256 + """ + + ca = os.path.join(self.beat_path, + "..", + "testing", + "environments", + "docker", + "elasticsearch", + "pki", + "ca", + "ca.crt") + + self.render_config_template( + elasticsearch={ + "hosts": self.get_elasticsearch_url_ssl(), + "username": "admin", + "password": "changeme", + "ssl.certificate_authorities": [ca], + "ssl.ca_sha256": "8hZS8gpciuzlu+7Xi0sdv8T7RKRRxG1TWKumUQsDam0=", + }, + ) + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) + proc.check_kill_and_wait() + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_sending_events_with_a_bad_sha256(self): + """ + Test Sending events while using ca pinning with a bad sha256 + """ + + ca = os.path.join(self.beat_path, + "..", + "testing", + "environments", + "docker", + "elasticsearch", + "pki", + "ca", + "ca.crt") + + self.render_config_template( + elasticsearch={ + "hosts": self.get_elasticsearch_url_ssl(), + "username": "admin", + "password": "changeme", + "ssl.certificate_authorities": [ca], + "ssl.ca_sha256": "not-good-sha", + }, + ) + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + self.wait_until(lambda: self.log_contains( + "provided CA certificate pins doesn't match any of the certificate authorities used to validate the certificate")) + proc.check_kill_and_wait() diff --git a/metricbeat/docs/autodiscover-aws-ec2-config.asciidoc b/metricbeat/docs/autodiscover-aws-ec2-config.asciidoc new file mode 100644 index 00000000000..42f9497c2aa --- /dev/null +++ b/metricbeat/docs/autodiscover-aws-ec2-config.asciidoc @@ -0,0 +1,25 @@ +{beatname_uc} supports templates for modules: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------------- +metricbeat.autodiscover: + providers: + - type: aws_ec2 + period: 1m + credential_profile_name: elastic-beats + templates: + - condition: + equals: + aws.ec2.tags.service: "mysql" + config: + - module: mysql + metricsets: ["status", "galera_status"] + period: 10s + hosts: ["root:password@tcp(${data.aws.ec2.public.ip}:3306)/"] + username: root + password: password +------------------------------------------------------------------------------------- + +This autodiscover provider takes our standard AWS credentials options. +With this configuration, `mysql` metricbeat module will be launched for all EC2 +instances that have `service: mysql` as a tag. diff --git a/metricbeat/docs/configuring-howto.asciidoc b/metricbeat/docs/configuring-howto.asciidoc index d532b8a5a3e..b282976c097 100644 --- a/metricbeat/docs/configuring-howto.asciidoc +++ b/metricbeat/docs/configuring-howto.asciidoc @@ -77,7 +77,9 @@ include::{libbeat-dir}/shared-env-vars.asciidoc[] :autodiscoverJolokia: :autodiscoverHints: +:autodiscoverAWSEC2: include::{libbeat-dir}/shared-autodiscover.asciidoc[] +:autodiscoverAWSEC2!: :standalone: include::{libbeat-dir}/yaml.asciidoc[] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3961c8901a1..bb92e2bf76e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -20992,7 +20992,7 @@ format: percent *`kubernetes.pod.cpu.usage.limit.pct`*:: + -- -CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if unlimited) +CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited) type: scaled_float @@ -22495,6 +22495,58 @@ type: long -- +[float] +=== storageclass + +kubernetes storage class metrics + + + +*`kubernetes.storageclass.name`*:: ++ +-- +Storage class name. + +type: keyword + +-- + +*`kubernetes.storageclass.provisioner`*:: ++ +-- +Volume provisioner for the storage class. + +type: keyword + +-- + +*`kubernetes.storageclass.reclaim_policy`*:: ++ +-- +Reclaim policy for dynamically created volumes + +type: keyword + +-- + +*`kubernetes.storageclass.volume_binding_mode`*:: ++ +-- +Mode for default provisioning and binding + +type: keyword + +-- + +*`kubernetes.storageclass.created`*:: ++ +-- +Storage class creation date + +type: date + +-- + [float] === system diff --git a/metricbeat/docs/images/metricbeat-tomcat-overview.png b/metricbeat/docs/images/metricbeat-tomcat-overview.png new file mode 100644 index 00000000000..a5ade0e8dfc Binary files /dev/null and b/metricbeat/docs/images/metricbeat-tomcat-overview.png differ diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 5d322520b0c..c15a096c398 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -99,6 +99,7 @@ metricbeat.modules: - state_service - state_persistentvolume - state_persistentvolumeclaim + - state_storageclass period: 10s hosts: ["kube-state-metrics:8080"] @@ -196,6 +197,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -236,6 +239,8 @@ include::kubernetes/state_service.asciidoc[] include::kubernetes/state_statefulset.asciidoc[] +include::kubernetes/state_storageclass.asciidoc[] + include::kubernetes/system.asciidoc[] include::kubernetes/volume.asciidoc[] diff --git a/metricbeat/docs/modules/kubernetes/state_storageclass.asciidoc b/metricbeat/docs/modules/kubernetes/state_storageclass.asciidoc new file mode 100644 index 00000000000..02aec973cd9 --- /dev/null +++ b/metricbeat/docs/modules/kubernetes/state_storageclass.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-kubernetes-state_storageclass]] +=== Kubernetes state_storageclass metricset + +experimental[] + +include::../../../module/kubernetes/state_storageclass/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kubernetes/state_storageclass/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 46a5f672939..f5c57b82477 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -36,7 +36,7 @@ metricbeat.modules: #password: "secret" # This can be used for service account based authorization: - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt ---- diff --git a/metricbeat/docs/modules/tomcat.asciidoc b/metricbeat/docs/modules/tomcat.asciidoc index 37910f2b8d4..e6e8476c34d 100644 --- a/metricbeat/docs/modules/tomcat.asciidoc +++ b/metricbeat/docs/modules/tomcat.asciidoc @@ -14,6 +14,12 @@ This module periodically fetches JMX metrics from Apache Tomcat. === Compatibility The module has been tested with Tomcat 7.0.24 and 9.0.24. Other versions are expected to work. +[float] +== Dashboard +An overview dashboard for Kibana is already included: + +image::./images/metricbeat-tomcat-overview.png[] + [float] === Usage The Tomcat module requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 010c9c40b9f..94109f6face 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -121,7 +121,7 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.20+| .20+| |<> +.21+| .21+| |<> |<> |<> |<> @@ -139,6 +139,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> experimental[] |<> +|<> experimental[] |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | @@ -217,7 +218,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> -|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> beta[] |<> beta[] |<> beta[] diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go index b1cc337aa80..f0015d31b3b 100644 --- a/metricbeat/include/list_docker.go +++ b/metricbeat/include/list_docker.go @@ -51,6 +51,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_resourcequota" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_service" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_statefulset" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_storageclass" _ "github.com/elastic/beats/metricbeat/module/kubernetes/system" _ "github.com/elastic/beats/metricbeat/module/kubernetes/volume" ) diff --git a/metricbeat/mb/lightmetricset.go b/metricbeat/mb/lightmetricset.go index dee1d0afb7b..11e49cd5c77 100644 --- a/metricbeat/mb/lightmetricset.go +++ b/metricbeat/mb/lightmetricset.go @@ -18,6 +18,9 @@ package mb import ( + "fmt" + "net/url" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" @@ -83,7 +86,8 @@ func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error // At this point host parser was already run, we need to run this again // with the overriden defaults if registration.HostParser != nil { - base.hostData, err = registration.HostParser(base.module, base.host) + host := m.useHostURISchemeIfPossible(base.host, base.hostData.URI) + base.hostData, err = registration.HostParser(base.module, host) if err != nil { return nil, errors.Wrapf(err, "host parser failed on light metricset factory for '%s/%s'", m.Module, m.Name) } @@ -96,6 +100,18 @@ func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error return registration, nil } +// useHostURISchemeIfPossible method parses given URI to extract protocol scheme and prepend it to the host. +// It prevents from skipping protocol scheme (e.g. https) while executing HostParser. +func (m *LightMetricSet) useHostURISchemeIfPossible(host, uri string) string { + u, err := url.ParseRequestURI(uri) + if err == nil { + if u.Scheme != "" { + return fmt.Sprintf("%s://%s", u.Scheme, u.Host) + } + } + return host +} + // baseModule does the configuration overrides in the base module configuration // taking into account the light metric set default configurations func (m *LightMetricSet) baseModule(from Module) (*BaseModule, error) { diff --git a/metricbeat/mb/lightmodules_test.go b/metricbeat/mb/lightmodules_test.go index e7a933cc47a..287b7bfc671 100644 --- a/metricbeat/mb/lightmodules_test.go +++ b/metricbeat/mb/lightmodules_test.go @@ -20,6 +20,7 @@ package mb import ( + "net/url" "testing" "time" @@ -194,22 +195,22 @@ func TestNewModuleFromConfig(t *testing.T) { "normal module": { config: common.MapStr{"module": "foo", "metricsets": []string{"bar"}}, expectedOption: "default", - expectedQuery: QueryParams{}, + expectedQuery: nil, }, "light module": { config: common.MapStr{"module": "service", "metricsets": []string{"metricset"}}, expectedOption: "test", - expectedQuery: QueryParams{}, + expectedQuery: nil, }, "light module default metricset": { config: common.MapStr{"module": "service"}, expectedOption: "test", - expectedQuery: QueryParams{}, + expectedQuery: nil, }, "light module override option": { config: common.MapStr{"module": "service", "option": "overriden"}, expectedOption: "overriden", - expectedQuery: QueryParams{}, + expectedQuery: nil, }, "light module with query": { config: common.MapStr{"module": "service", "query": common.MapStr{"param": "foo"}}, @@ -220,7 +221,7 @@ func TestNewModuleFromConfig(t *testing.T) { config: common.MapStr{"module": "service", "period": "42s"}, expectedOption: "test", expectedPeriod: 42 * time.Second, - expectedQuery: QueryParams{}, + expectedQuery: nil, }, "light module is broken": { config: common.MapStr{"module": "broken"}, @@ -237,7 +238,7 @@ func TestNewModuleFromConfig(t *testing.T) { "mixed module with standard and light metricsets": { config: common.MapStr{"module": "mixed", "metricsets": []string{"standard", "light"}}, expectedOption: "default", - expectedQuery: QueryParams{}, + expectedQuery: nil, }, "mixed module with unregistered and light metricsets": { config: common.MapStr{"module": "mixedbroken", "metricsets": []string{"unregistered", "light"}}, @@ -287,6 +288,96 @@ func TestNewModuleFromConfig(t *testing.T) { } } +func TestLightMetricSet_VerifyHostDataURI(t *testing.T) { + const hostEndpoint = "ceph-restful:8003" + const sampleHttpsEndpoint = "https://" + hostEndpoint + + r := NewRegister() + r.MustAddMetricSet("http", "json", newMetricSetWithOption, + WithHostParser(func(module Module, host string) (HostData, error) { + u, err := url.Parse(host) + if err != nil { + return HostData{}, err + } + return HostData{ + Host: u.Host, + URI: host, + }, nil + })) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + config, err := common.NewConfigFrom( + common.MapStr{ + "module": "httpextended", + "metricsets": []string{"extends"}, + "hosts": []string{sampleHttpsEndpoint}, + }) + require.NoError(t, err) + + _, metricSets, err := NewModule(config, r) + require.NoError(t, err) + require.Len(t, metricSets, 1) + + assert.Equal(t, hostEndpoint, metricSets[0].Host()) + assert.Equal(t, sampleHttpsEndpoint, metricSets[0].HostData().URI) +} + +func TestLightMetricSet_WithoutHostParser(t *testing.T) { + const sampleHttpsEndpoint = "https://ceph-restful:8003" + + r := NewRegister() + r.MustAddMetricSet("http", "json", newMetricSetWithOption) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + config, err := common.NewConfigFrom( + common.MapStr{ + "module": "httpextended", + "metricsets": []string{"extends"}, + "hosts": []string{sampleHttpsEndpoint}, + }) + require.NoError(t, err) + + _, metricSets, err := NewModule(config, r) + require.NoError(t, err) + require.Len(t, metricSets, 1) + + assert.Equal(t, sampleHttpsEndpoint, metricSets[0].Host()) + assert.Equal(t, sampleHttpsEndpoint, metricSets[0].HostData().URI) +} + +func TestLightMetricSet_VerifyHostDataURI_NonParsableHost(t *testing.T) { + const ( + postgresHost = "host1:5432" + postgresEndpoint = "postgres://user1:pass@host1:5432?connect_timeout=2" + postgresParsed = "connect_timeout=3 host=host1 password=pass port=5432 user=user1" + ) + + r := NewRegister() + r.MustAddMetricSet("http", "json", newMetricSetWithOption, + WithHostParser(func(module Module, host string) (HostData, error) { + return HostData{ + Host: postgresHost, + URI: postgresParsed, + }, nil + })) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + config, err := common.NewConfigFrom( + common.MapStr{ + "module": "httpextended", + "metricsets": []string{"extends"}, + "hosts": []string{postgresEndpoint}, + }) + require.NoError(t, err) + + _, metricSets, err := NewModule(config, r) + require.NoError(t, err) + require.Len(t, metricSets, 1) + + assert.Equal(t, postgresHost, metricSets[0].Host()) + assert.Equal(t, postgresParsed, metricSets[0].HostData().URI) +} + func TestNewModulesCallModuleFactory(t *testing.T) { logp.TestingSetup() diff --git a/metricbeat/mb/mb_test.go b/metricbeat/mb/mb_test.go index 3094e1588d6..f93721d8a81 100644 --- a/metricbeat/mb/mb_test.go +++ b/metricbeat/mb/mb_test.go @@ -81,9 +81,9 @@ func TestModuleConfig(t *testing.T) { err string }{ { - name: "missing required field", + name: "string value is not set on required field", in: map[string]interface{}{}, - err: "missing required field accessing 'module'", + err: "string value is not set accessing 'module'", }, { name: "valid config", @@ -97,7 +97,7 @@ func TestModuleConfig(t *testing.T) { Enabled: true, Period: time.Second * 10, Timeout: 0, - Query: QueryParams{}, + Query: nil, }, }, { diff --git a/metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml b/metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml new file mode 100644 index 00000000000..a56f13a3c7d --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml @@ -0,0 +1,4 @@ +default: true +input: + module: http + metricset: json diff --git a/metricbeat/mb/testdata/lightmodules/httpextended/module.yml b/metricbeat/mb/testdata/lightmodules/httpextended/module.yml new file mode 100644 index 00000000000..93ecaa59ba1 --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/httpextended/module.yml @@ -0,0 +1,3 @@ +name: httpextended +metricsets: +- extends diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 246a8d71bc3..01f17b9445c 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -482,6 +482,7 @@ metricbeat.modules: - state_service - state_persistentvolume - state_persistentvolumeclaim + - state_storageclass period: 10s hosts: ["kube-state-metrics:8080"] @@ -702,7 +703,7 @@ metricbeat.modules: #password: "secret" # This can be used for service account based authorization: - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt @@ -1199,6 +1200,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -1292,6 +1299,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index 6abf381e6da..10de0447725 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -39,6 +39,7 @@ - state_service - state_persistentvolume - state_persistentvolumeclaim + - state_storageclass period: 10s hosts: ["kube-state-metrics:8080"] diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index f242454913b..0325d0891e1 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -38,6 +38,7 @@ # - state_service # - state_persistentvolume # - state_persistentvolumeclaim +# - state_storageclass # period: 10s # hosts: ["kube-state-metrics:8080"] # add_metadata: true diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go index 7e772a7534b..a45bbe916f6 100644 --- a/metricbeat/module/kubernetes/fields.go +++ b/metricbeat/module/kubernetes/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKubernetes returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/kubernetes. func AssetKubernetes() string { - return "eJzsXU9z27iSv+dToHLKbHl02NraQw5bNeN5r54rmTyv7cwctrY0MNmSMCYBDgDa0fv0rwD+g0gABEVIdmzykIolsfuH7gbQ3QAaP6IH2H9ED+U9cAoSxDuEJJEZfETvP7Ufvn+HUAoi4aSQhNGP6H/eIYRQ9wOUg+QkUW9zyAAL+Ii2+B1CAqQkdCs+ov97L0T2/gK930lZvP9/9d2OcblOGN2Q7Ue0wZmAdwhtCGSp+KgZ/IgozqEHTz1yXygOnJVF/YkFnnqu6IbxHKuPEaYpEhJLIiRJBGIbVLBUoBxTvIUU3e8NPquagonGRIQLIoA/Am+/sYHyAOvJ76frK1QRNETZPIcibZ4+NBMeh79KEHKVZASoPPhJg/MB9k+Mp73vPGjVc6npIfgGSan02jASXhQcBCt5AvFw3FSUIUVW2n0Aorw/JQYX+QGMhBXxASBNFn1IslJI4BeaqShwAhetdH7w4noEfh8P1j/u7q7RgOTAMlkaURSa54DkkCeVQOVaMYqvhhqDZoEGLPpYUr5f85LGg/E7yB1wJHfQ8EClAIFSvkd9Rn0wD4T2uc1A8onQVI2uNfURleQFo3HHqIYk2mGaZmqUMoTiRdMfu2ciUYO6Jok2rNFMwDDxCFwQFtE0aoItimEz+xC05A4mt5kQmk5iI9xnnoPcsYj2qDumheig0UxENMO2xX2qDduCswSEsHK0GaJtvjfpJUW5EpAMvm9opqy8z/rj3qAhl9dfkYCE0bSPrOOUQ874Xk3rJAUqV/f7zjMb8s0Y3Vq+rPyyj8j18gGqn9WPEKGo4VljGIP4SLgscXZOhDXLMYCbVKxYAXSVsHIw+o1CO2D9pczvgasRVxFEG5JB+wPG3WoUEnMJaQSjua0MBglCE9BDTG3cDQ9rB1CBQDTrb+fVkmtvf1WKVQE8ASpJBqv/cLaQ3f8JiU0B1RfrKXJo+nwDAuUk4azuTqiD49aJrRmizGfqx48rKfMyw5I8ArKx8kGbb7wNNE1Jz1AN/VEggvwLqp4dU9NTQCsEk9RqQPZpNcaAdIBxoooNmKfQsCLvwSAKRgU8q3orCFP0OwR9egWbKIM1PAQaQ8U1FDupodMf36aahllnmioNsvLxd/J2TLVN4gNhgSxZll6T4zl5Eb0Fa+7GZJZhCTTZH2PJNm2JhuCFMlGFoPqbVI6TOSeNQopnQi0mOl0w92XyAPKsU07NGu2IkGzLcY4qEG6woa7EFBQNzUqToco7jefQYaGmI1x9GAbmGfTYoQ7XZFJyrsax+bK7opuMbHcywNQZ3fKSUkK3UUOVbvxM9KSl3kY1I39WGWSSriq5RxnJu6R/rU2BsNRcrOxxmRK5gkeXIqay1/SQpmdvb8WQg4IGaUSeDck+826uoRITOm+Nw5BuSy/KEoeOLNeS5PZUbopl/4uRhM2tIogGBI30SvAsPpahvP6KSoG3YBGEq9kmFP2usx/aAPmoHjSScRvhceJjDEwmlkG5z8YxljTPiHzN57I1OiX1S8ahFj3F1DlhHaDFlCmxuECPAg4EWxkFpCMMW1gshVVhnZM6VCLBGaTrTcaw64dNyFFHOTHaoKSLBcINTfU32+i0kGQSZxo7wlnGEizxfQbqPW9jM5IT+f21NoUNoZBW8NvsezcMflCfOCWCyAaVVL8LqX0BL2Pb8PzxSKs+s61ywzds4mCEHzHJsD0JNX9AckXCKKTnjYXTKFzXWjptU1GCC5wQuVeur516O6LWv3z90qksOVwyarB7/VLRQ3q4UIgaCdwrFfPmdrv3jiJOYnfaBrp+4myOsRDCwe9yxEKlGIUActhlfEDaNCyADtewoqWO3sZA3bfAkWW407nSL0sglRiczX3hfuWvBvqJrqVD/+jFe5chbZ7hYNYG4fYxTQnxwTYF9Kr6yM3trb+HNICfGH8gdCvAnQZ7DfL4vWomEiDD5FLgLWxwmVkSiVPSg3ZEXd5KsUEOPu2sif9k/Ex4NC8nqrb3MCY3Eff5vIWI4oYxqXeyiL2QkE8OLt6Gs2OXkul+v/UYzC6h2vN+vljsDDHGV0t0YWb2Ocsy4NXhh1kZ/suWWH2UIk5+/1m2oJ5zV/q5t7meeXur+jceuy84h7Bd1P9iNCLfK7rhWEheJrLkMCS+bOatmrNs5l028y6beQOasWzmtQNZNvMGY1w28y6beZfNvPM381q8zKnbe58Yf/irhNLucR4z9SnQoBzOasvd/On8c0Ww3VtXT+Y+X6KkG0KJ2EVxJ762xEJY4zSNYcO/N3pRBEcMOYVC7qLy1BRHu4/kJEp/7fiaO5g1dXtgxlJYJSpgTySzx9fHGC48kkR7EjF9YL1s0VD2GewOcCZ3MfaFd8xbqsieCDrFnnw/pwqPY6kqnN31wUKSu5HtmAQ4Bb4iYp1jIR05mXvGMsB9R2/s0PquO7WudU0E6vF410ejd6u+67OfkLC624FZeqPa/drkrEDNQ7pvtN/IHZYIc0BboMCxrGqFNHuF63H1gAOhKrBVwv3Ur1yCJiTD3Abm0LVX2pfV9Kq4IA4J46mo5N4anyQ5VJ8VmEuSlBnmlRDQDgvEEr0BPbUg1G9KnBcWlMPBxJf22xAu5LpmRR31OqZv7r1rAKp2ah6o46E+61uVedjj5IAUixE8XS5EDFbiKgwSvslwa/i1olNbAqRdcQDyCNQijoQV+7VkNgTdnIZFL9Rzp9686G40pVBwrRX2i24cyf1uX7RL7H6OOUicYomDzH5EHxUlhIVgCdGjzBORO69OfB3J3iWnz/DtIMQB95M/yNcBAlYqDjqBZkAY9Uv+pAnmmrOfpy6tE5exJokIRU87kuzqIfcJi27GsaJp8uDr6BVDfqsrhpgC8afdSxJxKeMrJX+VgHRymGyIchCYAcSSHGjToJBt1hmhDxHB3HxGHAoOQqGpq8m4BgRCH1n2COnagvFU40LD0yYX3wiBCxLfcn66vmrrzdTW41FX3MJDivdDXXxohHHcwYMag4eH6en6a0N5gujjdtivV7+M8DbDzzneu3GgTEcMy1my5SyZ44l9luyLsrfv+xjZsrfc9ix7y3tPvL3lyxbiHuBlC7Ed+LKF2LOFmIJUdhNtvObfXrXx3UAC5FHnaV202mwy57b1qEDMoXi+ufi02ZrXrZA7jqnIiZQvRyd3Vp20aehlv371BErz78tW/YkCWnbpd89AOG9hg76x0Ow4CNwHdY4T3B2ql3F2u8PjOr/d+jQldWZwjhm3Sa48wBOdxXfPCeMMxpigwB6OQlMkIT0dTUulXOXa450+a6DAmQO9ZTEGzC1oymD3BkVon4HaYPXgjM2cHHbB0u8yhb1EpNWzRKTd8z0p5LuLSN/EmtELWSUZwHqJRVGmFNt7UwX21JTa1kAR/SIoYZX1Iq+PLUtBPdgvskctZYbidbOjaw29jXTgQXdxN7m3aLh+7auGlVieBmuH7rDhlS8rVwJpD8oriegTgiNiKfAW1idbvaxABa+krs+Bxr2OapRo+LafE7UbZ0Y0rfkXtrbneiwlSY7eOe+qctJlkdMou+Rt1U2MffH9MiRzuAzItYLrVx+ZK7VDer7qHlMOt4zX9fCeqAys6TGtooen2/nHr2NqeUyq5BEZmbeGR2AFDw+kGdU7Qmp3hBvGlLodzqodx1n15Hod3kP+IbU6olTqmFqnIxYi7/H+6RU6Qo0zuDrHsbU5wrUaDnakasPEmhxxhpbwahyTa3Ecr0tLHY6jq3DEVWRY/Y2p1TdiqTK47sb0qhuTRWQjE1Zv4yi7sTmHY8U1jjl5HFhWo50O9zQJmpS8TB/Ke6jc9NpZ39PEmu8emdrKDETgzDAu/ts9Ta4VnBtFtneNGtu0H4xdiOdGN888nPgCrlZzY3JerxZznHFCH7tfrbeiWXD945zQbTS1f6lII4P2pCv0AiHO9F29ICcYwAjKs1iDvzFukxhkDUSyg7TM5pVINTIHLb0lbTDk8crSBoNDpkeyGSt+angmZRalYbe1lSIsJeSFHJJueLajQUS2qrPa6C7pmCUdMwZpSccs6ZiJiJZ0zJKOWdIxSzpmScdYMXir/1X8bbX/vBCm1P0bxGL9anvHTZLwn3D+sPRvNEWSIaCp0Rj7tBQIe05aYgIaTwfsI5rXI+yYfD2xYOmq4KDCFIVAFwvN58K4ZinqiKKaqAdBHSjF4NuQ8ra6lXitoHM6eLcWYxmfSQaI5/l0NhBBE8YAx8yUqctK3/UZv+z79I8uBzUQT3flOrHXrRMSyzLeAetih4V7t6C9Af1G+HYht83RjNCHuuLrBXrCROr/SOA5odh/hyLg1H0G3F49NxBlh1Azscv3wGNSEah7LxahEraDMr9HgKn4jFbCHlQNNcHM0t/vlYbQhxbVpa4yqZR2ybHYfWas+BknD2yzuUB/41yfBrsus+wCtf+tvx+qVj2Mt9pXI9CHS5YXGUhILzpJXGJKmbwpqWbB+AX65z9//USyDNIf6uavrB1lypmP0cLyevux66xDRde163iS2i+vv+raX6Ji6dF749SeBVLNDlJkZ3goJ9+5kJENiwWHRA0FH9F/r/4rBvIWS6BAfdjH4c3djumS+lnrkVVKPP1FUWMiqDd4VxvnR+sZNAp8ftyd2pq9+66TsAln9E92H8ulqahFcWgGqy/hLg26rHEMaPSXBecysNIxHMa6JLi9Z4Tw6UiggmWkR6k9dJEop3nGXSpdTqEipWIi0d1IPTASw+8Ua1GKAmg6OITuc40OuJvphMaEiIpZbXQ7y9UFrS1pfk8QchirFizZITFI9DcQnrCwls1uRyks5LqxgGg4lNB1QfkGBi+pvYPAtxOxV5RH2aeA04xQN+cxm/ulJtCyxhsJvO1SGknC9FUMXDmBG0wyQxMh//H/6Q71Uigyts9nXlRhDIwdwShjY4EtNR+Cu9tw9vhkRVpxsYUj3YxXZCTB4bHgUTgaLojQDZvoSaQgCPeUX5oVK/3SYez219QcO9QfRAHJnANzsTB2ZUUcejMOvNLzwTJ4BQArUus1BtFBVXyGgMwTsJEGh5h1sGMmZvwpj1mBvS7FbGY70AfJS7hAG5wJUFF5SR8oe6LuflPSeqbwGumsxIxGecDHNxjGjPaN47enC7DbetjmYV9/dN0UeBoBNaOyaoOpLSV1viLYhsyfK4T74jp7PRZ7top5VuQ1Wn8ZMGPZ5SS606fYT2Wapm5UjDSukJPC0Yf4+8XoWgEDF0RIoPKRZWUea7rqyKKKbjN3oQ1nuf7lj2qYhB89cxp8K4ATNdUeCOdUCYHfKqCKhCNV6+s/YfFMzcNaG9C31jG1EdUyBk4SxlN9nw0ztOPwCxjHW1gnGXbccR/A/bYigjSRNjUwsCwUEnC5LDTJMMlPZqZJhr8LY73+7dJjqVVj1nMY/ExoCmkjFjerOo24ru1nRt+46bL3TUeL3z+U3DQBO22cJCDEOu9vhJ/A4SdNAikSdh4n7GnXv12uXB3LPqXO6j2R6hoS+/11g4/DEwMK2dW1ldmOCbk+DUdF2sV2Ytg1jXEdHh1X6e2ES+s9mPXa+k2ztn4NVE1Oq9Xq2CX1mOjmRZpNRtKddYiJteVmw3sxRNvPxUGsnGVNsC7jM38oOGGy0ITqzlrGuAdpRp2b3eEN33V2sACObqo/bi3VoULzmM+Fy9+H46FS/XcqNnavK/ycSmj15Z76yryaE7rf67m6A6d3eXGW9c/5oYN1pHvwjS6xpLgps2zfcBuVprHdSB9Y+6tkBxflzhtaDJpRBpfTrQbe1Fj/V2MdWxPsS2kKgooDoRvGc0jRhx3mqZ6gBKQ/+A4Qxgk7DhvqXDqX/VuaJ7AwW1j1HPXqBfpDNfUP1dY/VGP/cMwfloYf0T5NTouyMj9cFBkBgSQbBqr+P92BrRoOSBIr41JT83WUM4eotzUiT0IlK4UEfpw7fkUlcIozdHXd2n0tBDs3+Fa9MCssbhrVEEO/fLl194OWpaOFxzB0BBgZw+n6HmeYJm6JBvD7zHCKfq7ptFblYDqnnzcNG9Bow0K65SoYP74tVxUFF/qGgYrbnDYxZoQVh3/YSPTmHfuIP1K0v5FSe3O65YUjhh6JJWzKLF4g0FCMFgn4hDaWSRo6Onc7Q4TttfnoA6gJvZo3b+sW9L3FM4QmB8Jrfa6jopMT+7NGiZDGnT3wEV1CRM8Qpgw2WfgANuA6h/3UejZCA8PZeVnqbpVsgH0Zam6UGwCsHfT03S+xxrvqIhmjOHaUQc9yfAhNmfEsF0G0G37dG12Xq9H7jxuQj+pBI1/d1ejLregWcsutD6aL9MortS8XgB8+ywXgYXjGC9dH3XxzuONmlkMyJ1UylIqxw8bKbbmRuX4C+99yI/NUAS03MnfPm7yR+WvgPcxnuPb4747LjvtQznEldOXk1WD+HQAA///hGQsL" + return "eJzsXU9z3LaSv/tToHxytpQ5bG3twYetSpT36qns+GklOzlsbU0wZM8MIhJgAFDyvE//CiD4Z0gABIeYsSxxDqlY0nT/0N0AuhuNxo/oAQ7v0UO5AU5BgniDkCQyg/fo7Yfmh2/fIJSCSDgpJGH0PfqfNwgh1P4BykFykqhvc8gAC3iPdvgNQgKkJHQn3qP/eytE9vYKvd1LWbz9f/W7PeNynTC6Jbv3aIszAW8Q2hLIUvFeM/gRUZxDD576yEOhOHBWFuYnFnjqc0O3jOdY/RhhmiIhsSRCkkQgtkUFSwXKMcU7SNHm0OGzMhS6aLqIcEEE8EfgzW9soDzAevL76fYGVQQ7oqw/xyKtP31oXXgc/ipByFWSEaDy6E9qnA9weGI87f3Og1Z9rjU9BF8hKZVea0bCi4KDYCVPIB6Ou4oypMhKuw9AlJtzYnCRH8BIWBEfANJk0bskK4UEfqWZigIncNVI5wcvrkfgm3iw/vH58y0akBxYJksjikLzHJAc8qQSqFwrRvHVYDBoFmjAoo8l5Yc1L2k8GL+D3ANHcg81D1QKECjlB9Rn1AfzQGif2wwkHwhN1epqqI+oJC8YjbtG1STRHtM0U6tURyheNP21eyYStahrkmjLas0ELBOPwAVhEU3DEGxQDIfZh6Ald7S5zYRQTxIb4T7zHOSeRbRHPTEtRAeDZiKiGTYj7lOt2RacJSCElaPNEG37fZdeUpQrAcng9zXNlJWbrL/uDQZyffsFCUgYTfvIWk455Iwf1LZOUqBytTm0ntmQb8bozvLLyi97j1xfPkL1s/ojRCiqeRoMYxAfCZclzi6J0LAcA7hNxYoVQFcJKwer3yi0I9afynwDXK24iiDakgyaP2DcrUYhMZeQRjCa+8pgkCA0Ab3EGOOueVgngAoEoll/s6+WXHv7q1KsCuAJUEkyWP2Hc4Rs8yckNgVUv1hPkUM952sQKCcJZ2Y6oRaOWye2YYgyn6kfP66kzMsMS/IIyMbKB22+8dbQNCW9Q9X0R4EI8i+oZnZMTU8BrRBMUmsHsk+rMRakI4wTVdyBeQ4NK/IeDKJgVMA3VW8FYYp+h6DPr+AuymAND4HGULGBYic1dPrj21Q9MOtOU6VBVj7+Tt6OrbZOfCAskCXL0htyPCcvordgzd10mWVYAk0Op1iyTVuiJnilTFQhqP5NKsepuyeNQopnQg0mOl0wmzJ5AHnRLcewRnsiJNtxnKMKhBtsqCsxBUVNs9JkqPLO4zm0WGjXEa5+GAbmG+ixRR2uyaTkXK1j82V3Q7cZ2e1lgKkzuuMlpYTuooYq7fqZ6E1LfRsZRv6sMsgkXVVyj7KSt0l/o02BsNRcrOxxmRK5gkeXIqay1/SQpmcfb8WQg4IGaUSeNck+83avoRITOu+MoyPdhl6UIw4dWa4lye2p3BTL/i9GEjb3iiAaEOykV4J38bEM5e0XVAq8A4sgXMPuQtHfdc5DGyAf1aNBMm4jPE58jEGXiWVR7rNxrCX1Z0S+3c91Y3RK6teMgxE9xdS5YR2hxZQpsbhAjwIOBFsZBaQjDBtYLIVVYd2TWlQiwRmk623GsOsP65DDRDkxxqCkiwXCNU31b7bVaSHJJM40doSzjCVY4k0G6nvewWYkJ/L7G20KW0IhreA32fd2GXynfuKUCCJbVFL9XUjtB3gZ24Xnj0dG9ZHtlBu+ZRMXI/yISYbtSaj5C5IrEkYhM28snEbhutbSaYaKElzghMiDcn3t1JsV1fzly5dOZcnhklGL3cuXil7Sw4VC1ErgPqmYt7fbvXcUcRP7rG2gnSfO4XQOQjj4XY5YqBSjEEAOu4wPSJuGBdDxGVa01NHrWKj7FjhyDHc+V/p5CaQSg3O4z9yv/LWDfqJr6dA/evbeZciYZziYxiDcPmZXQnxQpoBe1By5u7/3z5Aa8BPjD4TuBLjTYC9BHr9Xw0QCZJhcCryDLS4zSyJxSnrQjqjNWyk2yMGn2TXxn4xfCI/m5UTVzB7G5DZinc9riCjuGJO6kkUchIR8cnDxOpwdu5S67vdrj8HsEjKe97eLxS4QY3yxRBfdzD5nWQa8uvwwK8N/3RAzVyni5Pe/SQnqJavSL13meuHyVvXfeOw+4RzCqqj/xWhEvjd0y7GQvExkyWFIfCnmrYazFPMuxbxLMW/AMJZiXjuQpZg3GONSzLsU8y7FvPOLeS1e5tTy3ifGH/4qobR7nKdsfQo0KIezKrmbv51/rAg2tXVmM/f5EiXdEkrEPoo78aUhFsIap2kMG/691osiOGLIKRRyH5Wnpjg6fSQnUeZry7dbwayp2wMzlsIqUQF7Ipk9vj7FcOGRJNqTiOkD62OLmrLPYPeAM7mPURfeMm+oInsi6Bw1+X5OFR7HUVU4u9ujgyT3IJs1CXAKfEXEOsdCOnIyG8YywH1Hb+zS+r69ta51TQTq8XjTR6OrVd/02U9IWH3eQ7f1RlX9WuesQO1Dem40v5F7LBHmgHZAgWNZ9Qqpa4XNunrEgVAV2Crhfuh3LkETkmFuA3Po2ivt62p7VVwQh4TxVFRyb4xPkhyqnxWYS5KUGeaVENAeC8QSXYCeWhDqb0qcFxaUw8XEl/bbEi7k2rCijn4d04t7P9cA1Tg1D9TyUD/rW1X3ssfZASkWI3jaXIgYnMRVGCR8leHW8GtFx1gCpG1zAPII1CKOhBWHtWQ2BO2ehkUv1HOn3rzo7jSlUHCNFfabbpzI/fOhaI7Y/RxzkDjFEgeZ/Yg+KkoIC8ESoleZJyL3Xp34JpJ9Sk7f4ZtFiAPuJ3+QbwIEnFQcTQLNgDDql/xZE8yGs5+nbq0Tl7EmiQhFT3uS7M2S+4RFu+NY0dR58HX0jiG/mY4hXYH40+4liXiU8YWSv0pAOjlMtkQ5CKwDxJIcaNKgkG3XGaEPEcHcfUQcCg5CoTHdZFwLAqGPLHuEdG3BeK51oeZpk4tvhcAFiW85P93eNP1mjPV41BW38ZDi/WCaD40wjrt40M7i4WF6vvlaU54g+rgT9svNLyO8u+HnHO+9c6FMRwzLXbLlLpnjE/su2Sdlb9/3NbKlttz2WWrLe594teVLCXEP8FJCbAe+lBB7SogpSGU30dZr/vVFG98dJEAedZ7WRavJJnNuO48KxByK56uLT5OtedkK+cwxFTmR8vno5LNVJ00aeqnXrz6B0vz7Uqo/UUBLlX77GQjnNRTodw6aHReB+6AucYO7RfU87m63eFz3txufpqTODM4p6zbJlQd4prv47j1hnMEYExQ4w1FoiiRkpqNpqZSbXHu803cNFLhzoNcsxoC9BU1Z7F6hCO07UBOsHt2xmZPDLlj6Xaawl4i0+iwRafv5nhTy3UWkr+LM6JmckgxgPcemKFOa7b2qBntqS216oIh+ExTTWY9RQIyjnHHo/rEhrEhgDmP99yKfoi0HRj3Yz3LeLc2I4k3GkzsSvY6k4dF0cQ+5d7S4fulni5VYngYnjO7g4oUfPlcCaa7TK4noe4QjYinwDtZnO+OsQAWft64vgcZ92tpp5PD1MCe279ws0bTmP+va3P6xNC45ub7e1QulzTWnUWrpbT1QOtXz/WYlc7gMyDWC6/comSu1Y3q+HiBTrsCMd//w3rsM7Pwxre+HZ9r5169TOn5M6vcRGZm300dgnw8PpBk9PkI6fIQbxpTuHs7eHqdZ9eSuHt5WACEdPaL085jazSMWIm8TgOl9PEKNM7iHx6kdPMK1Gg52pLfDxM4dcZaW8J4dkzt2nK5LS7eOk3t1xFVkWJeOqT06YqkyuDvH9N4ck0VkIxPWleMku7E5h2MtOE65nxzYfKPZDg80CdqUvEwfyg1Ubrpx1g80sWbFR7a2MgMRuDOMi//+QJNbBedOke09tsa2zQ/Gns1zo5tnHk58AQ+wuTE5H2GLuc44oY+9wtY79yy4/uOc0F00tX+qSKMO7UkP7QVCnOm7ekFOMIARlBexBv9g3CYxyBqIZA9pmc1rpNrJHDT0lrTBkMcLSxsMrqKeyGasRWrHMymzKAO7N1aKsJSQF3JIuubZrAYR2arJaqO7pGOWdMwYpCUds6RjJiJa0jFLOmZJxyzpmCUdY8Xg7RFY8bd1CPRCmNIdcBCL9XvynbZJwn/C5cPSv9EUSYaApp3B2LelQNhz0hIT0HgmYB/RvBlhx+SbiQVLVwUHFaYoBLqlaD4Xxi1LUUsUGaIeBCZQisG3JuUddSNxo6BLOnj3FmMZ30kGiOf5dDYQQRvGAMfMlKnLSt/0GT/vV/dPbho1EE/7MDuxd7cTEssy3jXsYo+Fu1rQPoD+IHy1ys1wNCP0zvSFvUJPmEj9PxJ4Tij2v7QIOHXfFLf32A1E2SLUTOzyPfKYVATqrsUiVMJu0Az4BDAVn9F+2YPeol0ws/T3e6Uh9K5Bda17USqlXXMs9h8ZK37GyQPbbq/Q3zjXd8Zuyyy7Qs3/mt8PVas+jDfaVyvQu2uWFxlISK9aSVxjSpm8K6lmwfgV+uc/f/1AsgzSH8zwV9aJMuVmyGj7eV1+7LoRUdF1VR1PUvv17RfdIUxULD16r53ai0Ay7CBFdobHcvLdHhkpWCw4JGopeI/+e/VfMZA3WAIF6sM+Dm9uOaZL6hftWlYp8fzPSY2JwBR4V4Xzo10PagV+e9yt2urafdd92YQz+ifbxHJpKmpRHJrB6Uu4S4OuDY4Bjf6x4FwGVjodh9E0DrfPjBA+LQlUsIz0KDWXLhLlNM94caXNKVSkVEwk2nerB0bS8TvFWpSiAJoOrqr7XKMj7t10Qm1CRMWsNrqt5eq215Y0vycIOY5VC5bskRgk+msIT1hYm2s3qxQWcl1bQDQcSui67XwNg5fUPkHg65nYK8qj7FPAaUaom/OYzf1iCDSs8VYCb6aURpIw/WADV07gFpOso4mQ//H/0x3qpVBk7JDPfM6iszC2BKOsjQW2dIYInm7D3eODFWnFxRaOtDtekZEEh8eCJ+GouSBCt2yiJ5GCINzTpGlWrPRLi7GtrzEcW9TvRAHJnAtzsTC2zUcceutceKWXg9XhFQCsSK2PHUQHVfEZAuregI20OMTslh0zMeNPecwK7HXD5m62A72TvIQrtMWZ0Pe/S/pA2RN1z5uSmp3Ca6SzEjMa5REf32IYM9rvXL89X4DddM3uXvb1R9d1G6gRUDP6r9aYmoZTl2uV3ZH5twrhPrnuXo/Fno1ivilyg9bfLKxz7HIW3elb7Ocyza5uVIw0rpCzwtGX+Pst6xoBAxdESKDykWVlHmu7asmiim69d6EtZ7n+yx/VMgk/evY0+FoAJ2qrPRLOuRICv1VAFQlHqtY3f8LiGcPD2kHQd9YxdRDVMQZOEsZT/eoN62jH4RcwjnewTjLseAk/gPt9RQRpIk1qYGBZKCTgcllokmGSn81Mkwx/F8Z6+9u1x1KrwaznMPiZ0BTSWixuViaNuDb2M2Nu3LXZ+3qixZ8fSm6agJ02ThIQYp33C+EncPhJk0CKhJ3HGWfa7W/XK9fEsm+ps2ZPpO6HxP7K3eDH4YkBhezm1spsz4Rcn4ejIu1iOzHsmsbYhEen9YM749F6D6Y5W7+rz9ZvgarNabVanXqkHhPdvEizzki6sw4xsTbcbHivhmj7uTiIlbM0BE0bn/lLwRmThV2o7qxljNeSZvS52R+/A26ygwVwdFf9497SHSo0j/mtcPnncDxUav5OxcY2usPPuYRmngDVD+sZTmhz0Ht1C05XeXGW9e/5oaNzpA34VpdYUtyWWXaouY1Ks1NupC+s/VWyo+d05y0tHZpRFpfznQbeGaz/q7GOnQn2pTQFQcWB0C3jOaTo3R7zVG9QAtIffBcI44QdxwN1Hp3L/lvOE1h0R1jNHPXVK/SHGuofaqx/qMH+4dg/LAM/YXyanBZlZX64KDICAkk2DFT9/3QHtmo5IEmsjIuh5psoFw5R7w0iT0IlK4UEfpo7fkMlcIozdHPb2L0Rgp0bfK2+MCssrgdVE0O/fLp3z4OGpWOEpzB0BBgZw+l6gzNME7dEA/h9ZDhFPxs6jVU5mM6Z5/XABjSasJDuuArGTx/LTUXBhb5moOI2p02MGWHF4R82Er19x77ij7T2r6XUvK9u+cIJS4/EErZlFi8QqClGiwR8QhvLJA0dnc/7jgibx/XRO1AberVv3psR9L3FC4QmR8JrfK6TopMz+7OdFiG1O3vkI7qEiL5BmDIosvABrMG1Dvu59dwJDTrOzvNSd6PkDtjnoeZauQHAennXftp13qrXzcI+I9frCJfbASs4eySCMDqIPCcfOLWUWm+si8J1eqCPc9aWutVJXrumYqpfq3v2B4pzkmAVzZqtxJxl2I+/zInJhuiU5KwDgF9ZWhUmptUTtK1sCN0hTFNkuMR3Fo7UbncZmtmg30uKNQ+qx5c6reKjuACWy3STNGF5PKUpf3eXfV/4LaZX8TRMwvj533kbdCsbshl5oSzwuiAysr5mHIzAKaaOW909jM/jjZwzlUctb6B097YX/m7B8mj+8Wd5ND8Mz/gzDlFL0Y7rz2Y5JHNc8qFUOvVmVm7LK+bmEzj/llfMpwpoecW8/bzKV8y/BL5dfoGnwv/ueCC8D+USz6hXTp4B8+8AAAD//18V9+I=" } diff --git a/metricbeat/module/kubernetes/pod/_meta/fields.yml b/metricbeat/module/kubernetes/pod/_meta/fields.yml index ff7301ec552..1c46d4432b9 100644 --- a/metricbeat/module/kubernetes/pod/_meta/fields.yml +++ b/metricbeat/module/kubernetes/pod/_meta/fields.yml @@ -56,7 +56,7 @@ type: scaled_float format: percent description: > - CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if unlimited) + CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited) - name: memory type: group fields: diff --git a/metricbeat/module/kubernetes/state_storageclass/README.md b/metricbeat/module/kubernetes/state_storageclass/README.md new file mode 100644 index 00000000000..8223563fefc --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/README.md @@ -0,0 +1,47 @@ +# Kube-state-metrics/StorageClass + +This metricset connects to kube-state-metrics endpoint to retrieve and report Storage Class metrics. + +Interestingly enough kube-state-metrics does not repport annotations, we are unable to inform which storage class is default. We can consider enriching adding that info, or contributing back to kube-state-metrics to add annotations. + +## Version history + +- February 2020, first release using kube-state-metrics `v1.8.0`. + +## Configuration + +See the metricset documentation for the configuration reference. + +## Manual testing + +Probably your kubernetes cluster already has a storage class. You can add extra SCs: + +Example: + +```bash +kubectl apply -f - << EOF +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: beats-test-sc1 + labels: + testl1: value1 + testl2: value2 +provisioner: kubernetes.io/non-existing1 +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: beats-test-sc2 + labels: + testl3: value3 + testl4: value4 +provisioner: kubernetes.io/non-existing2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +EOF +``` + +Then run metricbeat pointing to the kube-state-metrics endpoint. diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/data.json b/metricbeat/module/kubernetes/state_storageclass/_meta/data.json new file mode 100644 index 00000000000..f8dcdfb1bdb --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/data.json @@ -0,0 +1,47 @@ +{ + "@timestamp": "2020-02-07T17:10:12.124Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "192.168.39.32:32042", + "type": "kubernetes" + }, + "kubernetes": { + "storageclass": { + "volume_binding_mode": "Immediate", + "name": "beats-test-sc2", + "created": "2020-02-07T10:14:28.000Z", + "provisioner": "kubernetes.io/non-existing2", + "reclaim_policy": "Delete" + }, + "labels": { + "testl3": "value3", + "testl4": "value4" + } + }, + "event": { + "duration": 13684387, + "dataset": "kubernetes.storageclass", + "module": "kubernetes" + }, + "ecs": { + "version": "1.4.0" + }, + "host": { + "name": "pl51" + }, + "agent": { + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "f4458304-dc2e-4831-be7a-45cd34126fd2", + "hostname": "tamag8", + "id": "e9952df4-592e-4102-84a6-ad0b333b3b98" + }, + "metricset": { + "name": "state_storageclass", + "period": 10000 + } +} diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/docs.asciidoc b/metricbeat/module/kubernetes/state_storageclass/_meta/docs.asciidoc new file mode 100644 index 00000000000..5ad3f81cfdb --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +The `state_storageclass` metricset for kubernetes storage class related metrics from `kube-state-metrics`. + diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/fields.yml b/metricbeat/module/kubernetes/state_storageclass/_meta/fields.yml new file mode 100644 index 00000000000..2f3adac5747 --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/fields.yml @@ -0,0 +1,21 @@ +- name: storageclass + type: group + description: > + kubernetes storage class metrics + release: experimental + fields: + - name: name + type: keyword + description: Storage class name. + - name: provisioner + type: keyword + description: Volume provisioner for the storage class. + - name: reclaim_policy + type: keyword + description: Reclaim policy for dynamically created volumes + - name: volume_binding_mode + type: keyword + description: Mode for default provisioning and binding + - name: created + type: date + description: Storage class creation date diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 new file mode 100644 index 00000000000..0a7712e74b7 --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 @@ -0,0 +1,11 @@ +# HELP kube_storageclass_info Information about storageclass. +# TYPE kube_storageclass_info gauge +kube_storageclass_info{storageclass="test_storageclass",provisioner="kubernetes.io/rbd",reclaimPolicy="Delete",volumeBindingMode="Immediate"} 1 +# HELP kube_storageclass_created Unix creation timestamp +# TYPE kube_storageclass_created gauge +kube_storageclass_created{storageclass="test_storageclass"} 1.501569018e+09 +# HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels. +# TYPE kube_storageclass_labels gauge +kube_storageclass_labels{storageclass="test_storageclass",label_foo="bar"} 1 + + diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected new file mode 100644 index 00000000000..cd0595a14f4 --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected @@ -0,0 +1,27 @@ +[ + { + "RootFields": {}, + "ModuleFields": { + "labels": { + "foo": "bar" + } + }, + "MetricSetFields": { + "created": "2017-08-01T06:30:18.000Z", + "name": "test_storageclass", + "provisioner": "kubernetes.io/rbd", + "reclaim_policy": "Delete", + "volume_binding_mode": "Immediate" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.storageclass", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + } +] \ No newline at end of file diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected new file mode 100644 index 00000000000..346cc2cf104 --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected @@ -0,0 +1,27 @@ +[ + { + "RootFields": {}, + "ModuleFields": { + "labels": { + "addonmanager_kubernetes_io_mode": "EnsureExists" + } + }, + "MetricSetFields": { + "created": "2020-01-07T16:11:11.000Z", + "name": "standard", + "provisioner": "k8s.io/minikube-hostpath", + "reclaim_policy": "Delete", + "volume_binding_mode": "Immediate" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.storageclass", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + } +] \ No newline at end of file diff --git a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go new file mode 100644 index 00000000000..356485d49a1 --- /dev/null +++ b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go @@ -0,0 +1,89 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package state_storageclass + +import ( + p "github.com/elastic/beats/metricbeat/helper/prometheus" + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + mb.Registry.MustAddMetricSet("kubernetes", "state_storageclass", + NewStorageClassMetricSet, + mb.WithHostParser(p.HostParser)) +} + +// StorageClassMetricSet is a prometheus based MetricSet that fetches +// and reports kube-state-metrics storage class metrics +type StorageClassMetricSet struct { + mb.BaseMetricSet + prometheus p.Prometheus + mapping *p.MetricsMapping +} + +// NewStorageClassMetricSet returns a prometheus based metricset for Storage classes +func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { + prometheus, err := p.NewPrometheusClient(base) + if err != nil { + return nil, err + } + + return &StorageClassMetricSet{ + BaseMetricSet: base, + prometheus: prometheus, + mapping: &p.MetricsMapping{ + Metrics: map[string]p.MetricMap{ + "kube_storageclass_info": p.InfoMetric(), + "kube_storageclass_labels": p.ExtendedInfoMetric( + p.Configuration{ + StoreNonMappedLabels: true, + NonMappedLabelsPlacement: mb.ModuleDataKey + ".labels", + MetricProcessingOptions: []p.MetricOption{p.OpLabelKeyPrefixRemover("label_")}, + }), + "kube_storageclass_created": p.Metric("created", p.OpUnixTimestampValue()), + }, + Labels: map[string]p.LabelMap{ + "storageclass": p.KeyLabel("name"), + "provisioner": p.Label("provisioner"), + "reclaimPolicy": p.Label("reclaim_policy"), + "volumeBindingMode": p.Label("volume_binding_mode"), + }, + }, + }, nil +} + +// Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as +// module rooted fields at the event that gets reported +func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { + events, err := m.prometheus.GetProcessedMetrics(m.mapping) + if err != nil { + m.Logger().Error(err) + reporter.Error(err) + return + } + + for _, event := range events { + event[mb.NamespaceKey] = "storageclass" + reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil)) + if !reported { + m.Logger().Debug("error trying to emit event") + return + } + } + return +} diff --git a/filebeat/scripts/generator/module/main.go b/metricbeat/module/kubernetes/state_storageclass/state_storageclass_test.go similarity index 55% rename from filebeat/scripts/generator/module/main.go rename to metricbeat/module/kubernetes/state_storageclass/state_storageclass_test.go index 737650d918d..524121ce6fa 100644 --- a/filebeat/scripts/generator/module/main.go +++ b/metricbeat/module/kubernetes/state_storageclass/state_storageclass_test.go @@ -15,32 +15,27 @@ // specific language governing permissions and limitations // under the License. -package main +// +build !integration + +package state_storageclass import ( - "flag" - "fmt" - "os" + "testing" - "github.com/elastic/beats/filebeat/generator/module" + "github.com/elastic/beats/metricbeat/helper/prometheus/ptest" ) -func main() { - moduleName := flag.String("module", "", "Name of the module") - modulePath := flag.String("path", ".", "Path to the generated fileset") - beatsPath := flag.String("beats_path", ".", "Path to elastic/beats") - flag.Parse() - - if *moduleName == "" { - fmt.Println("Missing parameter: module") - os.Exit(1) - } - - err := module.Generate(*moduleName, *modulePath, *beatsPath) - if err != nil { - fmt.Printf("Cannot generate module: %v\n", err) - os.Exit(2) - } - - fmt.Println("New module was generated, now you can start creating filesets by create-fileset command.") +func TestEventMapping(t *testing.T) { + ptest.TestMetricSet(t, "kubernetes", "state_storageclass", + ptest.TestCases{ + { + MetricsFile: "../_meta/test/ksm.v1.8.0", + ExpectedFile: "./_meta/test/ksm.v1.8.0.expected", + }, + { + MetricsFile: "./_meta/test/ksm.unit.v1.8.0", + ExpectedFile: "./_meta/test/ksm.unit.v1.8.0.expected", + }, + }, + ) } diff --git a/metricbeat/module/postgresql/_meta/kibana/7/dashboard/Metricbeat-postgresql-overview.json b/metricbeat/module/postgresql/_meta/kibana/7/dashboard/Metricbeat-postgresql-overview.json index 29b02299363..f9a3ba57b50 100644 --- a/metricbeat/module/postgresql/_meta/kibana/7/dashboard/Metricbeat-postgresql-overview.json +++ b/metricbeat/module/postgresql/_meta/kibana/7/dashboard/Metricbeat-postgresql-overview.json @@ -19,9 +19,11 @@ }, "panelsJSON": [ { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Query Latency" + }, "gridData": { - "h": 12, + "h": 13, "i": "1", "w": 24, "x": 0, @@ -30,10 +32,12 @@ "panelIndex": "1", "panelRefName": "panel_0", "title": "Query Latency", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Database Transactions" + }, "gridData": { "h": 12, "i": "2", @@ -44,24 +48,28 @@ "panelIndex": "2", "panelRefName": "panel_1", "title": "Database Transactions", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Fileblock IO" + }, "gridData": { "h": 10, "i": "3", "w": 24, "x": 0, - "y": 18 + "y": 19 }, "panelIndex": "3", "panelRefName": "panel_2", "title": "Fileblock IO", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Rows Fetched/Returned" + }, "gridData": { "h": 6, "i": "4", @@ -72,10 +80,12 @@ "panelIndex": "4", "panelRefName": "panel_3", "title": "Rows Fetched/Returned", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Rows Inserted/Deleted/Updated" + }, "gridData": { "h": 9, "i": "5", @@ -86,24 +96,28 @@ "panelIndex": "5", "panelRefName": "panel_4", "title": "Rows Inserted/Deleted/Updated", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Conflict/Deadlock Rates" + }, "gridData": { "h": 11, "i": "6", "w": 24, "x": 0, - "y": 28 + "y": 29 }, "panelIndex": "6", "panelRefName": "panel_5", "title": "Conflict/Deadlock Rates", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Database Filter" + }, "gridData": { "h": 6, "i": "7", @@ -114,10 +128,12 @@ "panelIndex": "7", "panelRefName": "panel_6", "title": "Database Filter", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Top Queries" + }, "gridData": { "h": 10, "i": "8", @@ -127,13 +143,15 @@ }, "panelIndex": "8", "panelRefName": "panel_7", - "title": "Query Calls Count", - "version": "7.3.0" + "title": "Top Queries", + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Local block cache stats" + }, "gridData": { - "h": 8, + "h": 9, "i": "9", "w": 12, "x": 24, @@ -142,12 +160,14 @@ "panelIndex": "9", "panelRefName": "panel_8", "title": "Local block cache stats", - "version": "7.3.0" + "version": "7.3.1" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "title": "Shared block cache stats" + }, "gridData": { - "h": 8, + "h": 9, "i": "10", "w": 12, "x": 36, @@ -156,7 +176,7 @@ "panelIndex": "10", "panelRefName": "panel_9", "title": "Shared block cache stats", - "version": "7.3.0" + "version": "7.3.1" } ], "timeRestore": false, @@ -220,8 +240,8 @@ } ], "type": "dashboard", - "updated_at": "2019-08-19T14:53:17.203Z", - "version": "WzE1NjQsMV0=" + "updated_at": "2020-02-06T12:05:20.696Z", + "version": "WzQzNTYsMV0=" }, { "attributes": { @@ -253,12 +273,17 @@ "id": "919c5570-b796-11e9-8ed3-ef1959e6b366", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "series": [ { "axis_position": "right", "chart_type": "line", "color": "rgba(0,156,224,1)", - "fill": 0, + "fill": "0.1", + "filter": { + "language": "kuery", + "query": "not postgresql.statement.query.text : (\"BEGIN;\" or \"begin\" or \"commit\" or \"end\" or \"END;\" or \"SELECT * FROM pg_stat_statements\" or \"SELECT * FROM pg_stat_database\" or \"SELECT * FROM pg_stat_bgwriter\" or \"SELECT * FROM pg_stat_activity\")" + }, "formatter": "number", "id": "919c5571-b796-11e9-8ed3-ef1959e6b366", "label": "Query Latency", @@ -284,11 +309,13 @@ ], "point_size": 0, "separate_axis": 0, + "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", "terms_field": "postgresql.statement.query.text", "terms_order_by": "919c7c80-b796-11e9-8ed3-ef1959e6b366", "terms_size": "10", + "type": "timeseries", "value_template": "{{value}}" } ], @@ -303,12 +330,12 @@ }, "id": "fbfa67e0-b796-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:40:30.561Z", - "version": "WzExNjcsMV0=" + "updated_at": "2020-02-06T11:59:01.903Z", + "version": "WzQzNTAsMV0=" }, { "attributes": { @@ -340,6 +367,7 @@ "id": "7af01590-b797-11e9-8816-2992f1df7a62", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "series": [ { "axis_position": "right", @@ -372,6 +400,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}}" }, { @@ -405,6 +434,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}}" } ], @@ -419,12 +449,12 @@ }, "id": "d733c630-b797-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:40:44.158Z", - "version": "WzExNzMsMV0=" + "updated_at": "2020-02-05T15:49:18.236Z", + "version": "WzQzMTIsMV0=" }, { "attributes": { @@ -438,7 +468,7 @@ } } }, - "title": "Fileblock IO [Metricbeat PostgreSQL] ECS", + "title": "Fileblock IO Rate [Metricbeat PostgreSQL] ECS", "uiStateJSON": {}, "version": 1, "visState": { @@ -456,6 +486,7 @@ "id": "fbc27280-b797-11e9-b46b-4f80f005c4a5", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "series": [ { "axis_position": "right", @@ -488,6 +519,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}} ms" }, { @@ -521,6 +553,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}} ms" } ], @@ -529,18 +562,18 @@ "time_field": "@timestamp", "type": "timeseries" }, - "title": "Fileblock IO [Metricbeat PostgreSQL] ECS", + "title": "Fileblock IO Rate [Metricbeat PostgreSQL] ECS", "type": "metrics" } }, "id": "570973a0-b798-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:41:16.761Z", - "version": "WzExODgsMV0=" + "updated_at": "2020-02-05T19:01:54.390Z", + "version": "WzQzMjQsMV0=" }, { "attributes": { @@ -590,6 +623,7 @@ "id": "a6981ed0-b798-11e9-a598-8baa89257193", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "series": [ { "axis_position": "right", @@ -600,7 +634,7 @@ "language": "kuery", "query": "" }, - "formatter": "number", + "formatter": "'0.0 a'", "id": "a6981ed1-b798-11e9-a598-8baa89257193", "label": "Rows Returned", "line_width": 2, @@ -635,6 +669,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}}" }, { @@ -669,6 +704,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}}" } ], @@ -683,12 +719,12 @@ }, "id": "66d67200-b799-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:40:15.889Z", - "version": "WzExNTgsMV0=" + "updated_at": "2020-02-06T11:56:03.315Z", + "version": "WzQzNDYsMV0=" }, { "attributes": { @@ -720,6 +756,7 @@ "id": "fc474800-b799-11e9-bfa6-bd2fe13c0445", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "series": [ { "axis_position": "right", @@ -832,12 +869,12 @@ }, "id": "20931ef0-b79a-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:41:36.559Z", - "version": "WzExOTYsMV0=" + "updated_at": "2020-02-06T12:01:45.205Z", + "version": "WzQzNTMsMV0=" }, { "attributes": { @@ -869,6 +906,7 @@ "id": "6c90db30-b79a-11e9-a8f0-d7983cd3d871", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "series": [ { "axis_position": "right", @@ -883,7 +921,7 @@ { "field": "postgresql.database.conflicts", "id": "6c90db32-b79a-11e9-a8f0-d7983cd3d871", - "type": "max" + "type": "avg" }, { "field": "6c90db32-b79a-11e9-a8f0-d7983cd3d871", @@ -901,6 +939,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}}" }, { @@ -916,7 +955,7 @@ { "field": "postgresql.database.deadlocks", "id": "6c90db36-b79a-11e9-a8f0-d7983cd3d871", - "type": "max" + "type": "avg" }, { "field": "6c90db36-b79a-11e9-a8f0-d7983cd3d871", @@ -934,6 +973,7 @@ "separate_axis": 0, "split_mode": "everything", "stacked": "none", + "type": "timeseries", "value_template": "{{value}}" } ], @@ -948,12 +988,12 @@ }, "id": "960ecdf0-b79a-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:42:02.764Z", - "version": "WzEyMDQsMV0=" + "updated_at": "2020-02-05T15:46:52.411Z", + "version": "WzQzMDgsMV0=" }, { "attributes": { @@ -1000,7 +1040,7 @@ }, "id": "98e6b0a0-b79b-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [ { @@ -1010,8 +1050,8 @@ } ], "type": "visualization", - "updated_at": "2019-08-09T11:39:56.460Z", - "version": "WzExNTEsMV0=" + "updated_at": "2020-01-22T13:56:51.268Z", + "version": "WzQwMjgsMV0=" }, { "attributes": { @@ -1034,6 +1074,11 @@ "axis_formatter": "number", "axis_position": "left", "axis_scale": "normal", + "background_color_rules": [ + { + "id": "d3cc2560-484b-11ea-a805-d7a3b4bc3300" + } + ], "bar_color_rules": [ { "id": "6da7d6e0-b902-11e9-9f00-7b1f283b2282" @@ -1045,9 +1090,18 @@ "language": "kuery", "query": "" }, + "gauge_color_rules": [ + { + "id": "d22727f0-484b-11ea-a805-d7a3b4bc3300" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, "id": "2bc5fea0-b902-11e9-8b8c-f99be54b4271", "index_pattern": "metricbeat-*", "interval": "auto", + "isModelInvalid": false, "pivot_id": "postgresql.statement.query.text", "pivot_type": "string", "series": [ @@ -1056,7 +1110,11 @@ "chart_type": "line", "color": "rgba(22,165,165,1)", "fill": 0, - "formatter": "number", + "filter": { + "language": "kuery", + "query": "not postgresql.statement.query.text : (\"BEGIN;\" or \"begin\" or \"commit\" or \"end\" or \"END;\" or \"SELECT * FROM pg_stat_statements\" or \"SELECT * FROM pg_stat_database\" or \"SELECT * FROM pg_stat_bgwriter\" or \"SELECT * FROM pg_stat_activity\")" + }, + "formatter": "'0a'", "id": "2bc5fea1-b902-11e9-8b8c-f99be54b4271", "label": "Number of times the query has been run", "line_width": 2, @@ -1064,7 +1122,7 @@ { "field": "postgresql.statement.query.calls", "id": "2bc5fea2-b902-11e9-8b8c-f99be54b4271", - "type": "max" + "type": "avg" } ], "point_size": 0, @@ -1073,13 +1131,15 @@ "stacked": "none", "terms_field": "postgresql.statement.query.text", "terms_order_by": "2bc5fea2-b902-11e9-8b8c-f99be54b4271", - "terms_size": "10", + "terms_size": "20", + "type": "timeseries", "value_template": "{{value}}" } ], "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "time_range_mode": "entire_time_range", "type": "top_n" }, "title": "Query Calls Count [Metricbeat PostgreSQL] ECS", @@ -1088,12 +1148,12 @@ }, "id": "147875b0-b903-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:40:59.993Z", - "version": "WzExODAsMV0=" + "updated_at": "2020-02-06T12:04:42.628Z", + "version": "WzQzNTUsMV0=" }, { "attributes": { @@ -1206,12 +1266,12 @@ }, "id": "0cb65170-b909-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:42:18.557Z", - "version": "WzEyMTAsMV0=" + "updated_at": "2020-01-22T13:56:51.268Z", + "version": "WzQwMzAsMV0=" }, { "attributes": { @@ -1322,13 +1382,13 @@ }, "id": "e2b28ce0-b908-11e9-a579-f5c0a5d81340", "migrationVersion": { - "visualization": "7.3.0" + "visualization": "7.4.2" }, "references": [], "type": "visualization", - "updated_at": "2019-08-09T11:42:42.659Z", - "version": "WzEyMTksMV0=" + "updated_at": "2020-01-22T13:56:51.268Z", + "version": "WzQwMzEsMV0=" } ], - "version": "7.3.0" + "version": "7.3.1" } diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index d322be32d5c..69f3219bbd2 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -6,6 +6,6 @@ #password: "secret" # This can be used for service account based authorization: - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt diff --git a/metricbeat/module/prometheus/collector/_meta/data.json b/metricbeat/module/prometheus/collector/_meta/data.json index da3bc943960..dba9f7771c4 100644 --- a/metricbeat/module/prometheus/collector/_meta/data.json +++ b/metricbeat/module/prometheus/collector/_meta/data.json @@ -23,4 +23,4 @@ "address": "127.0.0.1:55555", "type": "prometheus" } -} +} \ No newline at end of file diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 5f6fa9efce3..358dc58bf83 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -41,6 +41,7 @@ # - state_service # - state_persistentvolume # - state_persistentvolumeclaim +# - state_storageclass # period: 10s # hosts: ["kube-state-metrics:8080"] # add_metadata: true diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index f5dfb80f21e..21bc4c78800 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -9,6 +9,6 @@ #password: "secret" # This can be used for service account based authorization: - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index aa8c9399aa5..ad66e8fa18b 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -938,6 +938,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -1031,6 +1037,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/script/fix_permissions.sh b/script/fix_permissions.sh new file mode 100755 index 00000000000..fd65a7916b5 --- /dev/null +++ b/script/fix_permissions.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set +e +readonly LOCATION="${1?Please define the path where the fix permissions should run from}" + +if ! docker version ; then + echo "It requires Docker daemon to be installed and running" +else + set -e + # Change ownership of all files inside the specific folder from root/root to current user/group + docker run -v ${LOCATION}:/beat alpine:3.4 sh -c "find /beat -user 0 -exec chown -h $(id -u):$(id -g) {} \;" +fi diff --git a/testing/environments/docker/elasticsearch/pki/ca/ca.crt b/testing/environments/docker/elasticsearch/pki/ca/ca.crt new file mode 100644 index 00000000000..a18a84fd7b6 --- /dev/null +++ b/testing/environments/docker/elasticsearch/pki/ca/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAOshUH7Va8Kh1QeA4KgLw8dI29M4MA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTIwMDIwNzE2MzUzMFoXDTIzMDIwNjE2MzUzMFowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCv7Oq3uE0kO5Ij41U9M7ee +xOtprdA3joR68B32/SQ1FW9Igk3f/DTn8MqlAFexwzGAONPNIcj44W9KhVeT9aFA +qCKfS2no+V9aKds6wyEHY3sAmYICEHBMDor9KhPnIc8m/gl3TcGMKmouoHqFPNKE +irilBDUO7rs5w46lcbxJrHTlEA6xyQLT7+sJ4DswO/xeoemPTBa7vzkoVUyZ50/D +VSUulY4XtmQvmbe4Aa0p8sgLNzFAJRl3XqZMECwO2iJ9jFwKCUT4EbFW4aTQtylI +CBax+Cn79vKpp3gO1WVu1cdcQW3+ciAJyUydTsCA2zjGYZyzL84z7eCHW946WQWD +AgMBAAGjUzBRMB0GA1UdDgQWBBQZKfuW0o2yabRo9tosWldK43GDbjAfBgNVHSME +GDAWgBQZKfuW0o2yabRo9tosWldK43GDbjAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQAHeIJPwxvHTismlbFJKcCM3kr/ZblXguiFTmhqylqa8wFI +ke1xpTUrdfTAkD0ohmtPAUMPBkHeyHKzvxK7Blh230/lxybJNVSpfp7FQvj1EsmW +7FbIsKoj9MwJ2Lg5h6rnFA4t0bL3q74HV+vqpMoJDe92uX0GaSH/iYb+BfZ2El8m +QfANac0O+TE70i0++v/BzUAkqhJB3pG/3ziPzdFWlXf4iUG0YhMG4Ig5P/SvGz/V +MNc+uq3bh9xsNrtcm2S/pVdt/gdsujg9MTaoOr+maJPB/+LBrkZWtZcbUe++1+Z7 +32exp0eKNA0i90cc/Ayr79MOFDxdgI7baBnLPPa8 +-----END CERTIFICATE----- diff --git a/testing/environments/docker/elasticsearch/pki/ca/ca.key b/testing/environments/docker/elasticsearch/pki/ca/ca.key new file mode 100644 index 00000000000..48982cea36e --- /dev/null +++ b/testing/environments/docker/elasticsearch/pki/ca/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAr+zqt7hNJDuSI+NVPTO3nsTraa3QN46EevAd9v0kNRVvSIJN +3/w05/DKpQBXscMxgDjTzSHI+OFvSoVXk/WhQKgin0tp6PlfWinbOsMhB2N7AJmC +AhBwTA6K/SoT5yHPJv4Jd03BjCpqLqB6hTzShIq4pQQ1Du67OcOOpXG8Sax05RAO +sckC0+/rCeA7MDv8XqHpj0wWu785KFVMmedPw1UlLpWOF7ZkL5m3uAGtKfLICzcx +QCUZd16mTBAsDtoifYxcCglE+BGxVuGk0LcpSAgWsfgp+/byqad4DtVlbtXHXEFt +/nIgCclMnU7AgNs4xmGcsy/OM+3gh1veOlkFgwIDAQABAoIBAALZzfvoKqfZp0aZ +mnoBaopSGpZ90I/16UOsvG+SLpIFpOYB5o0ooxrXFhGSbdldlmHDifsa/wy5anpE +quSk6FYJ43W9XRv/XoIxh3HuU4yxGf8qfabW6VryKWJs2iG2tIqnNzQNuIMy9MGI +rDOYhrjLHq7d4JY7XCFVf+xCaZCwCb3yvZwVnrAqmPoeg2FrXmCVzqr1IpmwzJ0B +OfGWzi5THLm4/aGVUBfkvGURxsmwo3jGn0myr9oUkKczOKGEqvnlVuT9+ShURZp2 +tDU8zVRF0ksUNogUSfSNgWwpCYNBIqPOdxr7nT0+NEJ7b4R7/3LXEh/tRcuRNX+d +mjUMwbECgYEA/1MWpTdB9+9M7wyQasd29m1b0mkw+Jebdw+CuIK3XdPPGOfD17WO +sKZek3uK24DFGzRQf8tzHqzGvHncykAhb3oePVbfuhE5jt9bfgAOX8Naz6AK6Dmj +6+pJgXFTTNGL8JDojsIlabq4QH7oB02HoQ87GTr8IF4CjlJCHcyVB98CgYEAsGQO +uz0A1HdeuzbOP0+E86Ip03gcq66mVibXpy2qdMwEluxARW52XPKc8LKKI0QS4Qxk +giHHTQwPTLXJW9gM8v9/SQupQ/Vx8Zi3KjQ2ZAQoj6bGyDJ1P278GePJC4b0h/vG +F0sSUsmoEUGrLtq8Ofv3hDF6Ik247MQFi7i+Bt0CgYEAgP0kAqGw9SXzEw2g5CET +C5nh+qHj+KL3CqZOXxLCumcoSCfGe/KgPMRAIXgXhZ8/dOfwBy/sX8HfwRY7of3W +JnBmWIzMCD9tea2DlltG58BU33G2MO31z1iUfA2ZjMSMUyOSKZURu6F2Njcm15Gm +hIqiS7PN7jgwSGBsQIu7ercCgYEAh/nKJWrkbeVLgLTCD6okSpAzABLyvyJWlclB +q12Xrovr6dBbx2pdEk/wzdhEhuUeTKB6Bps1gV6PmMn2XLfTW6u8GrpDlODsIptg +b8dqOnW+MucVDBVhrzHGY8rmG93AOefMD/7ONEXCKvNdnDQAsA5eA2kExtb1fIer +4sbarn0CgYEAl1av+NOVduN1KrJXuZnNeN4KeNoYqJOS4s14Wk37GIujsrcE/m18 +BhZk0Al/oKZIDSuya5GGRhT+ndD8OWc4DEMWk2JnJdWKh20FfeM6UXVI46Sq3Won +vPDB6GYr45uRgtLvYeitLpXE5s9GmH8GyIV9om3TvDiceMXd/LbCodY= +-----END RSA PRIVATE KEY----- diff --git a/testing/environments/docker/elasticsearch/pki/elasticsearchssl/elasticsearchssl.crt b/testing/environments/docker/elasticsearch/pki/elasticsearchssl/elasticsearchssl.crt new file mode 100644 index 00000000000..4b373ea66a0 --- /dev/null +++ b/testing/environments/docker/elasticsearch/pki/elasticsearchssl/elasticsearchssl.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKjCCAhKgAwIBAgIUZWu3nanhrFaNe6kMhtsPM4neUCYwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjAwMjA3MTYzNjMxWhcNMjMwMjA2MTYzNjMxWjAbMRkwFwYD +VQQDExBlbGFzdGljc2VhcmNoc3NsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAl09yaI0HI3I8DmJ0UyRDedFNBFOfsFh7sYGGElj0h4H1kt8oEA3uYIH/ +oPUQ9Mkn30m+qccdQC6/pz/ZgkCfOckXtX1PVLEAK9MEqEwj6UU4uMgSIUTjXN22 +m/YedSJFtwGiQqFbCD1LijRLjlDCvHZ1W5M6XYzWoUN1y4MDZSD755TuluAp277j +6yuJIEw5SsQ/Nw4Coaqexy1ha7G/y3L+3t4SFrXaBqe+nM1xPDR0Is/p8iTdcdlu +kEFmbIqDGAPx2jvTRWYikL3MmR4u58AoIk0WqeGmLefxzV6jC6zsQGRnpmtz3jye +XHRfodf3crMZm+mw6FNPk4PJzZSsXwIDAQABo00wSzAdBgNVHQ4EFgQUmcNplxkS ++zHt5LWVM67Tzws8fBEwHwYDVR0jBBgwFoAUGSn7ltKNsmm0aPbaLFpXSuNxg24w +CQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAPpFdJpuwBXqBZezfQTTQFkKm +EHz3UDYKA3nHt2tcmFqAEXYx4cXaor5GG9YLThGWUp2iBXIyIzUZnpkM2wl/pIlz +8fMFxvtS6hQ2VwFDHAo2ht8ay7/vTrKcVvNL5NtPHjRlHhT94XiwYNpneiB6EMGP ++lTxWXSLpSnl0AnFdpLzPpS6DiaMHAPChAbDGK9i76D13sQBJZ/lgQiMmntEWsTr +0NNsjBk2xjMQAYs/eJXfENkAxvuzJTbQdJ1kMOvybONT4Lw8UIhoRpRY7EspwlI3 +encLBhcxYJjpzSPqdDQQRVXd4zUNFe4595LKEsm14mXaTy682HAe/HvN+yO7qw== +-----END CERTIFICATE----- diff --git a/testing/environments/docker/elasticsearch/pki/elasticsearchssl/elasticsearchssl.key b/testing/environments/docker/elasticsearch/pki/elasticsearchssl/elasticsearchssl.key new file mode 100644 index 00000000000..f374f10fa44 --- /dev/null +++ b/testing/environments/docker/elasticsearch/pki/elasticsearchssl/elasticsearchssl.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAl09yaI0HI3I8DmJ0UyRDedFNBFOfsFh7sYGGElj0h4H1kt8o +EA3uYIH/oPUQ9Mkn30m+qccdQC6/pz/ZgkCfOckXtX1PVLEAK9MEqEwj6UU4uMgS +IUTjXN22m/YedSJFtwGiQqFbCD1LijRLjlDCvHZ1W5M6XYzWoUN1y4MDZSD755Tu +luAp277j6yuJIEw5SsQ/Nw4Coaqexy1ha7G/y3L+3t4SFrXaBqe+nM1xPDR0Is/p +8iTdcdlukEFmbIqDGAPx2jvTRWYikL3MmR4u58AoIk0WqeGmLefxzV6jC6zsQGRn +pmtz3jyeXHRfodf3crMZm+mw6FNPk4PJzZSsXwIDAQABAoIBAB1rHu1g7hBgN3j8 +f21i0ZOvs++xaozYx0Pd0PlkPjbSd7KUnK9yZfRxkgfzXdaZ/ZyWM/HCetdtv2l/ +KoT+l3aeuHNa57+pokTjBDbMhvbltH+Itq2tPR9jJAvysD1J6pAIS0n1IUPa1wMJ +497JqPMHfQ3O9DwYE+rKuO5WjKRulUrL8K3OgHndLiHPZuUfIveSd6qux7wAebmD +OpWukVvYoC2k//Bgopdyg9VxVZtTg1SZlyFZ8wteDrbgF+eDMp9uIRddrvMUCwH4 ++GJOzkXxgkeOANjr5obMRjrr5hwoCE+RObCXAT3lx+nfCvYY5Lb72WWPQPEJ5ltP +xuxYY/ECgYEA71+DxCSUpxaK6THpJ10Z4FlTV0YAFfnMx9Jecn4CaJpQrWYFjLB7 +zkhlJWWyzPMc56+5olfcMEXHO9dT1/w3lFlJmRaS4yu/ZdPf2E6Pi6eXpeRYshj9 +NIq/pMCB1XxNogGzQA0AFBc+vw6Tx7LG+Bz/Yafi4SQN89I9v2SaeiMCgYEAodIE +epMZmVhlmrVzjPKcYtqWu464Sb3sHBwgnxvKcU1NUAUjTuzI9DwrJYgrA9NBcgHq +ckwbqiHNcej4MGFk7nN98U47eb+p6PAPNde7q42iNz2q7pKlNVml+Eg/wC2lhNah +N6K6S4wvTM6ujNIZGQ3DyKQC0tCMu+LnPxYYcpUCgYEAi9E2nfLgAVjheqR0k1GG +M8z5KRjyI+PtASqXkDiaH49DYIUe6LaNGkifC+EDN0MptwqlW3YGXwvi+8kiaB4i +OLyOiKTu11JOUaQYM7hvkBssMPHX/O8rtuz0U78+FvysO9zSXq85RILvW5mgKBz8 +qyAE632sv+TXYXuEJa8VrBECgYEAmAmh6aSh7aDPPc90NJ6R7pMgAjKy1Z4a48JN +qBBNYazWkfNx3Cq/GDIb+9R3Tc3D9KD2LNNzPqMpyvevkI1BboSWdQ0i9l3s/w3l +zJnYGvQk0DAhlKu1i22icac4NpDsreWWbZZ34Jliq5CZEXgo2pBDPhVTDc2iHLmw +uWZCLA0CgYAG99zukAD9iq2MyGiyuJ8dYU0dDvyaKcSkM3pfsgA4TpSlZljDKnAH +1VVPGB3pOHUYbcxsD2/1AJBlplvq8OVcrURuSXYl9PtwczJBgfSNNtSkHvMirWzo +q7eEeYCCs/VZUr9mY0nuzysq3ltiBW6tsdCn6d89ogs2WvseTlHZLg== +-----END RSA PRIVATE KEY----- diff --git a/testing/environments/docker/elasticsearch/pki/generate_pki.sh b/testing/environments/docker/elasticsearch/pki/generate_pki.sh new file mode 100755 index 00000000000..beb43d294ea --- /dev/null +++ b/testing/environments/docker/elasticsearch/pki/generate_pki.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Take the certificates and create a DER format and create a sha256 of it and encode it to base 64 +# https://www.openssl.org/docs/manmaster/man1/dgst.html +openssl x509 -in ca/ca.crt -pubkey -noout | openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl enc -base64 diff --git a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md index 4349863cb7b..01a9ea1d295 100644 --- a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md +++ b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md @@ -14,6 +14,13 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [0.8.2] + +### Fixed +- Fixed nonzero validator to not fail on nil array or slice. #147 +- Fixed nonzero validator to validate maps. +- Fixed required validator to validate maps. + ## [0.8.1] ### Fixed @@ -256,7 +263,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Introduced CHANGELOG.md for documenting changes to ucfg. -[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.8.1...HEAD +[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.8.2...HEAD +[0.8.2]: https://github.com/elastic/go-ucfg/compare/v0.8.1...v0.8.2 [0.8.1]: https://github.com/elastic/go-ucfg/compare/v0.8.0...v0.8.1 [0.8.0]: https://github.com/elastic/go-ucfg/compare/v0.7.0...v0.8.0 [0.7.0]: https://github.com/elastic/go-ucfg/compare/v0.6.5...v0.7.0 diff --git a/vendor/github.com/elastic/go-ucfg/error.go b/vendor/github.com/elastic/go-ucfg/error.go index a48690e30a8..d944bac66d9 100644 --- a/vendor/github.com/elastic/go-ucfg/error.go +++ b/vendor/github.com/elastic/go-ucfg/error.go @@ -98,6 +98,14 @@ var ( ErrRequired = errors.New("missing required field") ErrEmpty = errors.New("empty field") + + ErrArrayEmpty = errors.New("empty array") + + ErrMapEmpty = errors.New("empty map") + + ErrRegexEmpty = errors.New("regex value is not set") + + ErrStringEmpty = errors.New("string value is not set") ) // Error Classes diff --git a/vendor/github.com/elastic/go-ucfg/reify.go b/vendor/github.com/elastic/go-ucfg/reify.go index ad14f95baf7..11dea0a53ad 100644 --- a/vendor/github.com/elastic/go-ucfg/reify.go +++ b/vendor/github.com/elastic/go-ucfg/reify.go @@ -167,7 +167,7 @@ func reifyInto(opts *options, to reflect.Value, from *Config) Error { switch k { case reflect.Map: - return reifyMap(opts, to, from) + return reifyMap(opts, to, from, nil) case reflect.Struct: return reifyStruct(opts, to, from) case reflect.Slice, reflect.Array: @@ -183,7 +183,7 @@ func reifyInto(opts *options, to reflect.Value, from *Config) Error { return raiseInvalidTopLevelType(to.Interface(), opts.meta) } -func reifyMap(opts *options, to reflect.Value, from *Config) Error { +func reifyMap(opts *options, to reflect.Value, from *Config, validators []validatorTag) Error { parentFields := opts.activeFields defer func() { opts.activeFields = parentFields }() @@ -198,7 +198,7 @@ func reifyMap(opts *options, to reflect.Value, from *Config) Error { fields := from.fields.dict() if len(fields) == 0 { - if err := tryRecursiveValidate(to, opts, nil); err != nil { + if err := tryRecursiveValidate(to, opts, validators); err != nil { return raiseValidation(from.ctx, from.metadata, "", err) } return nil @@ -224,6 +224,9 @@ func reifyMap(opts *options, to reflect.Value, from *Config) Error { to.SetMapIndex(key, v) } + if err := runValidators(to.Interface(), validators); err != nil { + return raiseValidation(from.ctx, from.metadata, "", err) + } if err := tryValidate(to); err != nil { return raiseValidation(from.ctx, from.metadata, "", err) } @@ -321,7 +324,7 @@ func reifyGetField( } // Primitive types return early when it doesn't implement the Initializer interface. - if fieldType.Kind() != reflect.Map && fieldType.Kind() != reflect.Struct && !hasInitDefaults(fieldType) { + if fieldType.Kind() != reflect.Struct && !hasInitDefaults(fieldType) { if err := tryRecursiveValidate(to, opts.opts, opts.validators); err != nil { return raiseValidation(cfg.ctx, cfg.metadata, name, err) } @@ -479,7 +482,7 @@ func reifyMergeValue( if err != nil { return reflect.Value{}, raiseExpectedObject(opts.opts, val) } - return old, reifyMap(opts.opts, old, sub) + return old, reifyMap(opts.opts, old, sub, opts.validators) case reflect.Struct: sub, err := val.toConfig(opts.opts) diff --git a/vendor/github.com/elastic/go-ucfg/validator.go b/vendor/github.com/elastic/go-ucfg/validator.go index b56e8835dfd..8a72abf3b33 100644 --- a/vendor/github.com/elastic/go-ucfg/validator.go +++ b/vendor/github.com/elastic/go-ucfg/validator.go @@ -392,31 +392,53 @@ func validateRequired(v interface{}, name string) error { } return nil } - if err := validateNonEmpty(v, name); err != nil { - return ErrRequired + if err := validateNonEmptyWithAllowNil(v, name, false); err != nil { + return err } return nil } -func validateNonEmpty(v interface{}, _ string) error { +func validateNonEmpty(v interface{}, name string) error { + return validateNonEmptyWithAllowNil(v, name, true) +} + +func validateNonEmptyWithAllowNil(v interface{}, _ string, allowNil bool) error { if s, ok := v.(string); ok { if s == "" { - return ErrEmpty + return ErrStringEmpty } return nil } if r, ok := v.(regexp.Regexp); ok { if r.String() == "" { - return ErrEmpty + return ErrRegexEmpty } return nil } val := reflect.ValueOf(v) if val.Kind() == reflect.Array || val.Kind() == reflect.Slice { + if val.IsNil() { + if allowNil { + return nil + } + return ErrRequired + } + if val.Len() == 0 { + return ErrArrayEmpty + } + return nil + } + if val.Kind() == reflect.Map { + if val.IsNil() { + if allowNil { + return nil + } + return ErrRequired + } if val.Len() == 0 { - return ErrEmpty + return ErrMapEmpty } return nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 8ecfa1a8bd7..b98b14acbdb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -437,7 +437,7 @@ github.com/elastic/go-txfile/internal/vfs/osfs/osfstest github.com/elastic/go-txfile/pq github.com/elastic/go-txfile/txerr github.com/elastic/go-txfile/txfiletest -# github.com/elastic/go-ucfg v0.8.1 +# github.com/elastic/go-ucfg v0.8.2 github.com/elastic/go-ucfg github.com/elastic/go-ucfg/cfgutil github.com/elastic/go-ucfg/flag diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index fbf716b157d..51aaa89ec97 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -434,6 +434,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -527,6 +533,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 70c7d197a53..8f70d82821e 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -567,6 +567,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -660,6 +666,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/x-pack/dockerlogbeat/docs/configuration.asciidoc b/x-pack/dockerlogbeat/docs/configuration.asciidoc new file mode 100644 index 00000000000..7ffaa48154b --- /dev/null +++ b/x-pack/dockerlogbeat/docs/configuration.asciidoc @@ -0,0 +1,302 @@ +[[log-driver-configuration]] +[role="xpack"] +== {log-driver} configuration options + +++++ +Configuration options +++++ + +experimental[] + +Use the following options to configure the {log-driver-long}. You can +pass these options with the `--log-opt` flag when you start a container, or +you can set them in the `daemon.json` file for all containers. + +* <> +* <> +* <> +* <> +* <> + +[float] +=== Usage examples + +To set configuration options when you start a container: + +include::install.asciidoc[tag=log-driver-run] + +To set configuration options for all containers in the `daemon.json` file: + +include::install.asciidoc[tag=log-driver-daemon] + +For more examples, see <>. + +[float] +[[cloud-options]] +=== {ecloud} options + +[options="header"] +|===== +|Option | Description + +|`cloud.id` +|The Cloud ID found in the Elastic Cloud web console. This ID is +used to resolve the {stack} URLs when connecting to {ess} on {ecloud}. + +|`cloud.auth` +|The username and password combination for connecting to {ess} on {ecloud}. The +format is `"username:password"`. +|===== + +[float] +[[es-output-options]] +=== {es} output options + +// TODO: Add the following settings. Syntax is a little different so we might +// need to add deameon examples that show how to specify these settings: +// `output.elasticsearch.indices +// `output.elasticsearch.pipelines` + +[options="header"] +|===== +|Option |Default |Description + +|`output.elasticsearch.hosts` +|`"localhost:9200"` +|The list of {es} nodes to connect to. Specify each node as a `URL` or +`IP:PORT`. For example: `http://192.0.2.0`, `https://myhost:9230` or +`192.0.2.0:9300`. If no port is specified, the default is `9200`. + +|`output.elasticsearch.protocol` +|`http` +|The protocol (`http` or `https`) that {es} is reachable on. If you specify a +URL for `hosts`, the value of `protocol` is overridden by whatever scheme you +specify in the URL. + +|`output.elasticsearch.username` +| +|The basic authentication username for connecting to {es}. + +|`output.elasticsearch.password` +| +|The basic authentication password for connecting to {es}. + +|`output.elasticsearch.index` +| +|A {beats-ref}/config-file-format-type.html#_format_string_sprintf[format string] +value that specifies the index to write events to when you're using daily +indices. For example: +"dockerlogs-%{+yyyy.MM.dd}"+. + +3+|*Advanced:* + +|`output.elasticsearch.backoff.init` +|`1s` +|The number of seconds to wait before trying to reconnect to {es} after +a network error. After waiting `backoff.init` seconds, the {log-driver} +tries to reconnect. If the attempt fails, the backoff timer is increased +exponentially up to `backoff.max`. After a successful connection, the backoff +timer is reset. + +|`output.elasticsearch.backoff.max` +|`60s` +|The maximum number of seconds to wait before attempting to connect to +{es} after a network error. + +|`output.elasticsearch.bulk_max_size` +|`50` +|The maximum number of events to bulk in a single {es} bulk API index request. +Specify 0 to allow the queue to determine the batch size. + +|`output.elasticsearch.compression_level` +|`0` +|The gzip compression level. Valid compression levels range from 1 (best speed) +to 9 (best compression). Specify 0 to disable compression. Higher compression +levels reduce network usage, but increase CPU usage. + +|`output.elasticsearch.escape_html` +|`false` +|Whether to escape HTML in strings. + +|`output.elasticsearch.headers` +| +|Custom HTTP headers to add to each request created by the {es} output. Specify +multiple header values for the same header name by separating them with a comma. + +|`output.elasticsearch.loadbalance` +|`false` +|Whether to load balance when sending events to multiple hosts. The load +balancer also supports multiple workers per host (see +`output.elasticsearch.worker`.) + +|`output.elasticsearch.max_retries` +|`3` +|The number of times to retry publishing an event after a publishing failure. +After the specified number of retries, the events are typically dropped. Specify +0 to retry indefinitely. + +|`output.elasticsearch.parameters` +| +| A dictionary of HTTP parameters to pass within the URL with index operations. + +|`output.elasticsearch.path` +| +|An HTTP path prefix that is prepended to the HTTP API calls. This is useful for +cases where {es} listens behind an HTTP reverse proxy that exports the API under +a custom prefix. + +|`output.elasticsearch.pipeline` +| +|A {beats-ref}/config-file-format-type.html#_format_string_sprintf[format string] +value that specifies the {ref}/ingest.html[ingest node pipeline] to write events +to. + +|`output.elasticsearch.proxy_url` +| +|The URL of the proxy to use when connecting to the {es} servers. Specify a +`URL` or `IP:PORT`. + +|`output.elasticsearch.timeout` +|`90` +|The HTTP request timeout in seconds for the {es} request. + +|`output.elasticsearch.worker` +|`1` +|The number of workers per configured host publishing events to {es}. Use with +load balancing mode (`output.elasticsearch.loadbalance`) set to `true`. Example: +If you have 2 hosts and 3 workers, in total 6 workers are started (3 for each +host). + +|===== + + +[float] +[[ls-output-options]] +=== {ls} output options + +[options="header"] +|===== +|Option | Default | Description + +|`output.logstash.hosts` +|`"localhost:5044"` +|The list of known {ls} servers to connect to. If load balancing is +disabled, but multiple hosts are configured, one host is selected randomly +(there is no precedence). If one host becomes unreachable, another one is +selected randomly. If no port is specified, the default is `5044`. + +|`output.logstash.index` +| +|The index root name to write events to. For example +"dockerlogs"+ generates ++"dockerlogs-{version}"+ indices. + +3+|*Advanced:* + +|`output.logstash.backoff.init` +|`1s` +|The number of seconds to wait before trying to reconnect to {ls} after +a network error. After waiting `backoff.init` seconds, the {log-driver} +tries to reconnect. If the attempt fails, the backoff timer is increased +exponentially up to `backoff.max`. After a successful connection, the backoff +timer is reset. + +|`output.logstash.backoff.max` +|`60s` +|The maximum number of seconds to wait before attempting to connect to +{ls} after a network error. + +|`output.logstash.bulk_max_size` +|`2048` +|The maximum number of events to bulk in a single {ls} request. Specify 0 to +allow the queue to determine the batch size. + +|`output.logstash.compression_level` +|`0` +|The gzip compression level. Valid compression levels range from 1 (best speed) +to 9 (best compression). Specify 0 to disable compression. Higher compression +levels reduce network usage, but increase CPU usage. + +|`output.logstash.escape_html` +|`false` +|Whether to escape HTML in strings. + +|`output.logstash.loadbalance` +|`false` +|Whether to load balance when sending events to multiple {ls} hosts. If set to +`false`, the driver sends all events to only one host (determined at random) and +switches to another host if the selected one becomes unresponsive. + +|`output.logstash.pipelining` +|`2` +|The number of batches to send asynchronously to {ls} while waiting for an ACK +from {ls}. Specify 0 to disable pipelining. + +|`output.logstash.proxy_url` +| +|The URL of the SOCKS5 proxy to use when connecting to the {ls} servers. The +value must be a URL with a scheme of `socks5://`. You can embed a +username and password in the URL (for example, +`socks5://user:password@socks5-proxy:2233`). + +|`output.logstash.proxy_use_local_resolver` +|`false` +|Whether to resolve {ls} hostnames locally when using a proxy. If `false`, +name resolution occurs on the proxy server. + +|`output.logstash.slow_start` +|`false` +|When enabled, only a subset of events in a batch are transferred per +transaction. If there are no errors, the number of events per transaction +is increased up to the bulk max size (see `output.logstash.bulk_max_size`). +On error, the number of events per transaction is reduced again. + +|`output.logstash.timeout` +|`30` +|The number of seconds to wait for responses from the {ls} server before +timing out. + +|`output.logstash.ttl` +|`0` +|Time to live for a connection to {ls} after which the connection will be +re-established. Useful when {ls} hosts represent load balancers. Because +connections to {ls} hosts are sticky, operating behind load balancers can lead +to uneven load distribution across instances. Specify a TTL on the connection +to distribute connections across instances. Specify 0 to disable this feature. +This option is not supported if `output.logstash.pipelining` is set. + +|`output.logstash.worker` +|`1` +|The number of workers per configured host publishing events to {ls}. Use with +load balancing mode (`output.logstash.loadbalance`) set to `true`. Example: +If you have 2 hosts and 3 workers, in total 6 workers are started (3 for each +host). + +|===== + +[float] +[[kafka-output-options]] +=== Kafka output options + +// TODO: Add kafka output options here. + +// NOTE: The following annotation renders as: "Coming in a future update. This +// documentation is a work in progress." + +coming[a future update. This documentation is a work in progress] + +Need the docs now? See the +{filebeat-ref}/kafka-output.html[Kafka output docs] for {filebeat}. +The {log-driver} supports most of the same options, just make sure you use +the fully qualified setting names. + +[float] +[[redis-output-options]] +=== Redis output options + +// TODO: Add Redis output options here. + +coming[a future update. This documentation is a work in progress] + +Need the docs now? See the +{filebeat-ref}/redis-output.html[Redis output docs] for {filebeat}. +The {log-driver} supports most of the same options, just make sure you use +the fully qualified setting names. diff --git a/x-pack/dockerlogbeat/docs/index.asciidoc b/x-pack/dockerlogbeat/docs/index.asciidoc new file mode 100644 index 00000000000..ca3f2cd4007 --- /dev/null +++ b/x-pack/dockerlogbeat/docs/index.asciidoc @@ -0,0 +1,25 @@ +:libbeat-dir: {docdir}/../../../libbeat/docs +:log-driver: Elastic Logging Plugin +:log-driver-long: Elastic Logging Plugin for Docker +:log-driver-alias: elastic-logging-plugin +:docker-version: Engine API 1.25 + += {log-driver} for Docker + +include::{libbeat-dir}/version.asciidoc[] + +include::{asciidoc-dir}/../../shared/versions/stack/{source_branch}.asciidoc[] + +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + +include::overview.asciidoc[] + +include::install.asciidoc[] + +include::configuration.asciidoc[] + +include::usage.asciidoc[] + +include::troubleshooting.asciidoc[] + +include::limitations.asciidoc[] diff --git a/x-pack/dockerlogbeat/docs/install.asciidoc b/x-pack/dockerlogbeat/docs/install.asciidoc new file mode 100644 index 00000000000..23babe7a3f6 --- /dev/null +++ b/x-pack/dockerlogbeat/docs/install.asciidoc @@ -0,0 +1,116 @@ +[[log-driver-installation]] +[role="xpack"] +== Install and configure the {log-driver} + +++++ +Install and configure +++++ + +experimental[] + +[float] +=== Before you begin + +Make sure your system meets the following prerequisites: + +* Docker: {docker-version} or later +* {stack}: Version 7.6.0 or later + +[float] +=== Step 1: Install the {log-driver} plugin + +// TODO: Test the following commands when the driver is available on docker hub. + +1. Install the plugin. You can install it from the Docker store (recommended), +or build and install the plugin from source in the +https://github.com/elastic/beats[beats] GitHub repo. ++ +*To install from the Docker store:* ++ +["source","sh",subs="attributes"] +---- +docker plugin install elastic/{log-driver-alias}:{version} +---- ++ +*To build and install from source:* ++ +{beats-devguide}/beats-contributing.html#setting-up-dev-environment[Set up your +development environment] as described in the _Beats Developer Guide_ then run: ++ +[source,shell] +---- +cd x-pack/dockerlogbeat +mage BuildAndInstall +---- + +2. If necessary, enable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin enable elastic/{log-driver-alias}:{version} +---- + +3. Verify that the plugin is installed and enabled: ++ +[source,shell] +---- +docker plugin ls +---- ++ +The output should say something like: ++ +["source","sh",subs="attributes"] +---- +ID NAME DESCRIPTION ENABLED +c2ff9d2cf090 elastic/{log-driver-alias}:{version} A beat for docker logs true +---- + +[float] +=== Step 2: Configure the {log-driver} + +You can set configuration options for a single container, or for all containers +running on the host. See <> for a list of +supported configuration options. + +*To configure a single container:* + +Pass configuration options at run time when you start the container. For +example: + +// tag::log-driver-run[] +["source","sh",subs="attributes"] +---- +docker run --log-driver=elastic/{log-driver-alias}:{version} \ + --log-opt output.elasticsearch.hosts="https://myhost:9200" \ + --log-opt output.elasticsearch.username="myusername" \ + --log-opt output.elasticsearch.password="mypassword" \ + --log-opt output.elasticsearch.index="elastic-log-driver-%{+yyyy.MM.dd}" \ + -it debian:jessie /bin/bash +---- +// end::log-driver-run[] + +*To configure all containers running on the host:* + +Set configuration options in the Docker `daemon.json` configuration file. For +example: + +// tag::log-driver-daemon[] +[source,json,subs="attributes"] +---- +{ + "log-driver" : "elastic/{log-driver-alias}:{version}", + "log-opts" : { + "output.elasticsearch.hosts" : "https://myhost:9200", + "output.elasticsearch.username" : "myusername", + "output.elasticsearch.password" : "mypassword", + "output.elasticsearch.index" : "elastic-log-driver-%{+yyyy.MM.dd}" + } +} +---- +// end::log-driver-daemon[] + +NOTE: The default location of the `daemon.json` file varies by platform. On +Linux, the default location is `/etc/docker/daemon.json`. For more information, +see the +https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file[Docker +docs]. diff --git a/x-pack/dockerlogbeat/docs/limitations.asciidoc b/x-pack/dockerlogbeat/docs/limitations.asciidoc new file mode 100644 index 00000000000..7ccb9cfeacb --- /dev/null +++ b/x-pack/dockerlogbeat/docs/limitations.asciidoc @@ -0,0 +1,13 @@ +[[log-driver-limitations]] +[role="xpack"] +== Known problems and limitations + +experimental[] + +This release of the {log-driver} has the following known problems and +limitations: + +* Spool to disk (beta) is not supported. +* Complex config options can't be easily represented via `--log-opts`. +* Mapping templates and other assets that are normally installed by the +{beats} setup are not available. diff --git a/x-pack/dockerlogbeat/docs/overview.asciidoc b/x-pack/dockerlogbeat/docs/overview.asciidoc new file mode 100644 index 00000000000..78bc7692865 --- /dev/null +++ b/x-pack/dockerlogbeat/docs/overview.asciidoc @@ -0,0 +1,19 @@ +[[log-driver-overview]] +[role="xpack"] +== {log-driver} overview + +++++ +Overview +++++ + +experimental[] + +The {log-driver} is a Docker plugin that sends container logs to the +https://www.elastic.co/elastic-stack[{stack}], where you can search, analyze, +and visualize the data in real time. + +The {log-driver} is built on top of the https://www.elastic.co/beats[{beats}] +platform and supports many of the features and outputs supported by the +{beats} shippers. + +Beat users: see <>. diff --git a/x-pack/dockerlogbeat/docs/troubleshooting.asciidoc b/x-pack/dockerlogbeat/docs/troubleshooting.asciidoc new file mode 100644 index 00000000000..478269ff6ec --- /dev/null +++ b/x-pack/dockerlogbeat/docs/troubleshooting.asciidoc @@ -0,0 +1,42 @@ +[[log-driver-troubleshooting]] +[role="xpack"] +== Troubleshooting + +experimental[] + +You can set the debug level to capture debugging output about the {log-driver}. +To set the debug level: + +1. Disable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin disable elastic/{log-driver-alias}:{version} +---- + +2. Set the debug level: ++ +["source","sh",subs="attributes"] +---- +docker plugin set elastic/{log-driver-alias}:{version} LOG_DRIVER_LEVEL=debug +---- ++ +Where valid settings for `LOG_DRIVER_LEVEL` are `debug`, `info`, `warning`, or +`error`. + +3. Enable the plugin: ++ +["source","sh",subs="attributes"] +---- +docker plugin enable elastic/{log-driver-alias}:{version} +---- + +To view the logs: + +On Linux, the {log-driver} logs are written to the same location as other +docker logs, typically the system journal. + +On MacOS, locating the logs is more complicated. For more information, see +the +https://github.com/elastic/beats/tree/{branch}/x-pack/dockerlogbeat#debugging-on-macos[Debugging +on MacOS] section in the readme file. \ No newline at end of file diff --git a/x-pack/dockerlogbeat/docs/usage.asciidoc b/x-pack/dockerlogbeat/docs/usage.asciidoc new file mode 100644 index 00000000000..b2100435baf --- /dev/null +++ b/x-pack/dockerlogbeat/docs/usage.asciidoc @@ -0,0 +1,94 @@ +[[log-driver-usage-examples]] +== {log-driver} usage examples + +++++ +Usage examples +++++ + +experimental[] + +The following examples show common configurations for the {log-driver}. + +[float] +=== Send Docker logs to {es} + +*Docker run command:* + +["source","sh",subs="attributes"] +---- +docker run --log-driver=elastic/{log-driver-alias}:{version} \ + --log-opt output.elasticsearch.hosts="myhost:9200" \ + --log-opt output.elasticsearch.protocol="https" \ + --log-opt output.elasticsearch.username="myusername" \ + --log-opt output.elasticsearch.password="mypassword" \ + --log-opt output.elasticsearch.index="elastic-log-driver-%{+yyyy.MM.dd}" \ + -it debian:jessie /bin/bash +---- + +*Daemon configuration:* + +["source","json",subs="attributes"] +---- +{ + "log-driver" : "elastic/{log-driver-alias}:{version}", + "log-opts" : { + "output.elasticsearch.hosts" : "myhost:9200", + "output.elasticsearch.protocol" : "https", + "output.elasticsearch.username" : "myusername", + "output.elasticsearch.password" : "mypassword", + "output.elasticsearch.index" : "elastic-log-driver-%{+yyyy.MM.dd}" + } +} +---- + +[float] +=== Send Docker logs to {ess} on {ecloud} + +*Docker run command:* + +["source","sh",subs="attributes"] +---- +docker run --log-driver=elastic/{log-driver-alias}:{version} \ + --log-opt cloud.id="MyElasticStack:daMbY2VudHJhbDekZ2NwLmN4b3VkLmVzLmliJDVkYmQwtGJiYjs0NTRiN4Q5ODJmNGUwm1IxZmFkNjM5JDFiNjdkMDE4MTgxMTQzNTM5ZGFiYWJjZmY0OWIyYWE5" \ + --log-opt cloud.auth="myusername:mypassword" \ + --log-opt output.elasticsearch.index="elastic-log-driver-%{+yyyy.MM.dd}" \ + -it debian:jessie /bin/bash +---- + +*Daemon configuration:* + +["source","json",subs="attributes"] +---- +{ + "log-driver" : "elastic/{log-driver-alias}:{version}", + "log-opts" : { + "cloud.id" : "MyElasticStack:daMbY2VudHJhbDekZ2NwLmN4b3VkLmVzLmliJDVkYmQwtGJiYjs0NTRiN4Q5ODJmNGUwm1IxZmFkNjM5JDFiNjdkMDE4MTgxMTQzNTM5ZGFiYWJjZmY0OWIyYWE5", + "cloud.auth" : "myusername:mypassword", + "output.elasticsearch.index" : "elastic-log-driver-%{+yyyy.MM.dd}" + } +} +---- + +[float] +=== Send Docker logs to {ls} + +*Docker run command:* + +["source","sh",subs="attributes"] +---- +docker run --log-driver=elastic/{log-driver-alias}:{version} \ + --log-opt output.logstash.hosts="myhost:5044" \ + -it debian:jessie /bin/bash +---- + +*Daemon configuration:* + +["source","json",subs="attributes"] +---- +{ + "log-driver" : "elastic/{log-driver-alias}:{version}", + "log-opts" : { + "output.logstash.hosts" : "myhost:5044" + } +} +---- diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 06da5fe0465..6b8c21273eb 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -642,6 +642,14 @@ filebeat.modules: # can be added under this section. #input: + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + #ingress_controller: + # enabled: false + # + # # Set custom paths for the log files. If left empty, + # # Filebeat will choose the paths depending on your OS. + # #var.paths: + #------------------------------- Osquery Module ------------------------------- - module: osquery result: @@ -1736,6 +1744,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -1829,6 +1843,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 5a95675f649..a1baece249f 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -38,9 +38,6 @@ type Pagination struct { } func (c *config) Validate() error { - if c.Interval < 3600*time.Second && c.Interval != 0 { - return errors.New("httpjson input: interval must not be less than 3600 seconds - ") - } switch strings.ToUpper(c.HTTPMethod) { case "GET": break diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 77cd867494f..6c8ad1a98b6 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -192,6 +192,28 @@ func TestPOST(t *testing.T) { }) } +func TestRepeatedPOST(t *testing.T) { + m := map[string]interface{}{ + "http_method": "POST", + "http_request_body": map[string]interface{}{"test": "abc", "testNested": map[string]interface{}{"testNested1": 123}}, + "interval": 10 ^ 9, + } + runTest(t, m, func(input *httpjsonInput, out *stubOutleter, t *testing.T) { + group, _ := errgroup.WithContext(context.Background()) + group.Go(input.run) + + events, ok := out.waitForEvents(3) + if !ok { + t.Fatalf("Expected 3 events, but got %d.", len(events)) + } + input.Stop() + + if err := group.Wait(); err != nil { + t.Fatal(err) + } + }) +} + func TestRunStop(t *testing.T) { m := map[string]interface{}{ "http_method": "GET", diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 8ba0c6550de..e733648b32d 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -150,8 +150,12 @@ func (in *httpjsonInput) createHTTPRequest(ctx context.Context, ri *requestInfo) } // processHTTPRequest processes HTTP request, and handles pagination if enabled -func (in *httpjsonInput) processHTTPRequest(ctx context.Context, client *http.Client, req *http.Request, ri *requestInfo) error { +func (in *httpjsonInput) processHTTPRequest(ctx context.Context, client *http.Client, ri *requestInfo) error { for { + req, err := in.createHTTPRequest(ctx, ri) + if err != nil { + return err + } msg, err := client.Do(req) if err != nil { return errors.New("failed to do http request. Stopping input worker - ") @@ -224,10 +228,6 @@ func (in *httpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl if in.config.Pagination.ExtraBodyContent != nil { ri.ContentMap.Update(common.MapStr(in.config.Pagination.ExtraBodyContent)) } - req, err = in.createHTTPRequest(ctx, ri) - if err != nil { - return err - } continue } return nil @@ -274,11 +274,7 @@ func (in *httpjsonInput) run() error { if in.config.HTTPMethod == "POST" && in.config.HTTPRequestBody != nil { ri.ContentMap.Update(common.MapStr(in.config.HTTPRequestBody)) } - req, err := in.createHTTPRequest(ctx, ri) - if err != nil { - return err - } - err = in.processHTTPRequest(ctx, client, req, ri) + err = in.processHTTPRequest(ctx, client, ri) if err == nil && in.Interval > 0 { ticker := time.NewTicker(in.Interval) defer ticker.Stop() @@ -288,7 +284,8 @@ func (in *httpjsonInput) run() error { in.log.Info("Context done.") return nil case <-ticker.C: - err = in.processHTTPRequest(ctx, client, req, ri) + in.log.Info("Process another repeated request.") + err = in.processHTTPRequest(ctx, client, ri) if err != nil { return err } diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index d40cb140796..a8ffd9d6875 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -18,6 +18,8 @@ import ( // mage:import "github.com/elastic/beats/dev-tools/mage/target/common" + // mage:import generate + _ "github.com/elastic/beats/filebeat/scripts/mage/generate" ) func init() { diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index ab6a5e5bd44..e6052c2822c 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -777,6 +777,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -870,6 +876,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/x-pack/libbeat/autodiscover/providers/aws/aws.go b/x-pack/libbeat/autodiscover/providers/aws/aws.go index 34e92c6addb..413866acc9e 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/aws.go +++ b/x-pack/libbeat/autodiscover/providers/aws/aws.go @@ -3,3 +3,38 @@ // you may not use this file except in compliance with the Elastic License. package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" + "github.com/pkg/errors" +) + +// SafeString makes handling AWS *string types easier. +// The AWS lib never returns plain strings, always using pointers, probably for memory efficiency reasons. +// This is a bit odd, because strings are just pointers into byte arrays, however this is the choice they've made. +// This will return the plain version of the given string or an empty string if the pointer is null +func SafeString(str *string) string { + if str == nil { + return "" + } + + return *str +} + +// GetRegions makes DescribeRegions API call to list all regions from AWS +func GetRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error) { + input := &ec2.DescribeRegionsInput{} + req := svc.DescribeRegionsRequest(input) + output, err := req.Send(context.TODO()) + if err != nil { + err = errors.Wrap(err, "Failed DescribeRegions") + return + } + for _, region := range output.Regions { + completeRegionsList = append(completeRegionsList, *region.RegionName) + } + return +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/config.go b/x-pack/libbeat/autodiscover/providers/aws/config.go similarity index 52% rename from x-pack/libbeat/autodiscover/providers/aws/elb/config.go rename to x-pack/libbeat/autodiscover/providers/aws/config.go index 0eb8fb73c0c..e3a0873734e 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/config.go +++ b/x-pack/libbeat/autodiscover/providers/aws/config.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package elb +package aws import ( "time" @@ -10,31 +10,23 @@ import ( "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/libbeat/autodiscover/template" - "github.com/elastic/beats/libbeat/common" ) -// Config for the aws_elb autodiscover provider. +// Config for all aws autodiscover providers. type Config struct { - Type string `config:"type"` - - // Standard autodiscover fields. - - // Hints are currently not supported, but may be implemented in a later release - HintsEnabled bool `config:"hints.enabled"` - Builders []*common.Config `config:"builders"` - Appenders []*common.Config `config:"appenders"` - Templates template.MapperSettings `config:"templates"` + Type string `config:"type"` + Templates template.MapperSettings `config:"templates"` // Period defines how often to poll the AWS API. Period time.Duration `config:"period" validate:"nonzero,required"` // AWS Specific autodiscover fields - - Regions []string `config:"regions" validate:"required"` + Regions []string `config:"regions"` AWSConfig aws.ConfigAWS `config:",inline"` } -func defaultConfig() *Config { +// DefaultConfig for all aws autodiscover providers. +func DefaultConfig() *Config { return &Config{ Period: time.Minute, } diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/_meta/fields.yml b/x-pack/libbeat/autodiscover/providers/aws/ec2/_meta/fields.yml new file mode 100644 index 00000000000..d634a6697df --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: ec2_listener + title: "EC2 Listener" + description: > + AWS EC2 Listeners + short_config: false + release: experimental + fields: + - name: ec2_listener + type: group + description: > + Represents an AWS EC2 Listener, e.g. state of an EC2. diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/ec2.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/ec2.go new file mode 100644 index 00000000000..3a81acb4d5a --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/ec2.go @@ -0,0 +1,134 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" +) + +type ec2Instance struct { + ec2Instance ec2.Instance +} + +// toMap converts this ec2Instance into the form consumed as metadata in the autodiscovery process. +func (i *ec2Instance) toMap() common.MapStr { + architecture, err := i.ec2Instance.Architecture.MarshalValue() + if err != nil { + logp.Error(errors.Wrap(err, "MarshalValue failed for architecture: ")) + } + + m := common.MapStr{ + "image": i.toImage(), + "vpc": i.toVpc(), + "subnet": i.toSubnet(), + "private": i.toPrivate(), + "public": i.toPublic(), + "monitoring": i.toMonitoringState(), + "kernel": i.toKernel(), + "state": i.stateMap(), + "architecture": architecture, + "root_device_name": awsauto.SafeString(i.ec2Instance.RootDeviceName), + } + + for _, tag := range i.ec2Instance.Tags { + m.Put("tags."+awsauto.SafeString(tag.Key), awsauto.SafeString(tag.Value)) + } + return m +} + +func (i *ec2Instance) instanceID() string { + return awsauto.SafeString(i.ec2Instance.InstanceId) +} + +func (i *ec2Instance) toImage() common.MapStr { + m := common.MapStr{} + m["id"] = awsauto.SafeString(i.ec2Instance.ImageId) + return m +} + +func (i *ec2Instance) toMonitoringState() common.MapStr { + monitoringState, err := i.ec2Instance.Monitoring.State.MarshalValue() + if err != nil { + logp.Error(errors.Wrap(err, "MarshalValue failed for monitoring state: ")) + } + + m := common.MapStr{} + m["state"] = monitoringState + return m +} + +func (i *ec2Instance) toPrivate() common.MapStr { + m := common.MapStr{} + m["ip"] = awsauto.SafeString(i.ec2Instance.PrivateIpAddress) + m["dns_name"] = awsauto.SafeString(i.ec2Instance.PrivateDnsName) + return m +} + +func (i *ec2Instance) toPublic() common.MapStr { + m := common.MapStr{} + m["ip"] = awsauto.SafeString(i.ec2Instance.PublicIpAddress) + m["dns_name"] = awsauto.SafeString(i.ec2Instance.PublicDnsName) + return m +} + +func (i *ec2Instance) toVpc() common.MapStr { + m := common.MapStr{} + m["id"] = awsauto.SafeString(i.ec2Instance.VpcId) + return m +} + +func (i *ec2Instance) toSubnet() common.MapStr { + m := common.MapStr{} + m["id"] = awsauto.SafeString(i.ec2Instance.SubnetId) + return m +} + +func (i *ec2Instance) toKernel() common.MapStr { + m := common.MapStr{} + m["id"] = awsauto.SafeString(i.ec2Instance.KernelId) + return m +} + +func (i *ec2Instance) toCloudMap() common.MapStr { + m := common.MapStr{} + availabilityZone := awsauto.SafeString(i.ec2Instance.Placement.AvailabilityZone) + m["availability_zone"] = availabilityZone + m["provider"] = "aws" + + // The region is just an AZ with the last character removed + m["region"] = availabilityZone[:len(availabilityZone)-1] + + instance := common.MapStr{} + instance["id"] = i.instanceID() + m["instance"] = instance + + instanceType, err := i.ec2Instance.InstanceType.MarshalValue() + if err != nil { + logp.Error(errors.Wrap(err, "MarshalValue failed for instance type: ")) + } + machine := common.MapStr{} + machine["type"] = instanceType + m["machine"] = machine + return m +} + +// stateMap converts the State part of the ec2 struct into a friendlier map with 'reason' and 'code' fields. +func (i *ec2Instance) stateMap() (stateMap common.MapStr) { + state := i.ec2Instance.State + stateMap = common.MapStr{} + nameString, err := state.Name.MarshalValue() + if err != nil { + logp.Error(errors.Wrap(err, "MarshalValue failed for instance state name: ")) + } + + stateMap["name"] = nameString + stateMap["code"] = state.Code + return stateMap +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/fetch.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/fetch.go new file mode 100644 index 00000000000..6f1e6fd38fb --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/fetch.go @@ -0,0 +1,230 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "context" + "sync" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" + "go.uber.org/multierr" + + "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" +) + +// fetcher is an interface that can fetch a list of ec2Instance objects without pagination being necessary. +type fetcher interface { + fetch(ctx context.Context) ([]*ec2Instance, error) +} + +// apiMultiFetcher fetches results from multiple clients concatenating their results together +// Useful since we have a fetcher per region, this combines them. +type apiMultiFetcher struct { + fetchers []fetcher +} + +func (amf *apiMultiFetcher) fetch(ctx context.Context) ([]*ec2Instance, error) { + fetchResults := make(chan []*ec2Instance) + fetchErr := make(chan error) + + // Simultaneously fetch all from each region + for _, f := range amf.fetchers { + go func(f fetcher) { + res, err := f.fetch(ctx) + if err != nil { + fetchErr <- err + } else { + fetchResults <- res + } + }(f) + } + + var results []*ec2Instance + var errs []error + + for pending := len(amf.fetchers); pending > 0; pending-- { + select { + case r := <-fetchResults: + results = append(results, r...) + case e := <-fetchErr: + errs = append(errs, e) + } + } + + return results, multierr.Combine(errs...) +} + +// apiFetcher is a concrete implementation of fetcher that hits the real AWS API. +type apiFetcher struct { + client ec2iface.ClientAPI +} + +func newAPIFetcher(clients []ec2iface.ClientAPI) fetcher { + fetchers := make([]fetcher, len(clients)) + for idx, client := range clients { + fetchers[idx] = &apiFetcher{client} + } + return &apiMultiFetcher{fetchers} +} + +// fetch attempts to request the full list of ec2Instance objects. +// It accomplishes this by fetching a page of EC2 instances, then one go routine +// per listener API request. Each page of results has O(n)+1 perf since we need that +// additional fetch per EC2. We let the goroutine scheduler sort things out, and use +// a sync.Pool to limit the number of in-flight requests. +func (f *apiFetcher) fetch(ctx context.Context) ([]*ec2Instance, error) { + var MaxResults int64 = 50 + + describeInstanceInput := &ec2.DescribeInstancesInput{MaxResults: &MaxResults} + req := f.client.DescribeInstancesRequest(describeInstanceInput) + + ctx, cancel := context.WithCancel(ctx) + ir := &fetchRequest{ + paginator: ec2.NewDescribeInstancesPaginator(req), + client: f.client, + taskPool: sync.Pool{}, + context: ctx, + cancel: cancel, + logger: logp.NewLogger("autodiscover-ec2-fetch"), + } + + // Limit concurrency against the AWS API by creating a pool of objects + // This is hard coded for now. The concurrency limit of 10 was set semi-arbitrarily. + for i := 0; i < 10; i++ { + ir.taskPool.Put(nil) + } + + return ir.fetch() +} + +// fetchRequest provides a way to get all pages from a +// ec2.DescribeInstancesPaginator and all listeners for the given EC2 instance. +type fetchRequest struct { + paginator ec2.DescribeInstancesPaginator + client ec2iface.ClientAPI + ec2Instances []*ec2Instance + errs []error + resultsLock sync.Mutex + taskPool sync.Pool + pendingTasks sync.WaitGroup + context context.Context + cancel func() + logger *logp.Logger +} + +func (p *fetchRequest) fetch() ([]*ec2Instance, error) { + p.dispatch(p.fetchAllPages) + + // Only fetch future pages when there are no longer requests in-flight from a previous page + p.pendingTasks.Wait() + + // Acquire the results lock to ensure memory + // consistency between the last write and this read + p.resultsLock.Lock() + defer p.resultsLock.Unlock() + + // Since everything is async we have to retrieve any errors that occurred from here + if len(p.errs) > 0 { + return nil, multierr.Combine(p.errs...) + } + + return p.ec2Instances, nil +} + +func (p *fetchRequest) fetchAllPages() { + // Keep fetching pages unless we're stopped OR there are no pages left + for { + select { + case <-p.context.Done(): + p.logger.Debug("done fetching EC2 instances, context cancelled") + return + default: + if !p.fetchNextPage() { + p.logger.Debug("fetched all EC2 instances") + return + } + p.logger.Debug("fetched EC2 instance") + } + } +} + +func (p *fetchRequest) fetchNextPage() (more bool) { + success := p.paginator.Next(p.context) + + if success { + for _, reservation := range p.paginator.CurrentPage().Reservations { + for _, instance := range reservation.Instances { + p.dispatch(func() { p.fetchInstances(instance) }) + } + } + } + + if p.paginator.Err() != nil { + p.recordErrResult(p.paginator.Err()) + } + + return success +} + +// dispatch runs the given func in a new goroutine, properly throttling requests +// with the taskPool and also managing the pendingTasks waitGroup to ensure all +// results are accumulated. +func (p *fetchRequest) dispatch(fn func()) { + p.pendingTasks.Add(1) + + go func() { + slot := p.taskPool.Get() + defer p.taskPool.Put(slot) + defer p.pendingTasks.Done() + + fn() + }() +} + +func (p *fetchRequest) fetchInstances(instance ec2.Instance) { + describeInstancesInput := &ec2.DescribeInstancesInput{InstanceIds: []string{awsauto.SafeString(instance.InstanceId)}} + req := p.client.DescribeInstancesRequest(describeInstancesInput) + listen := ec2.NewDescribeInstancesPaginator(req) + + if listen.Err() != nil { + p.recordErrResult(listen.Err()) + } + + for { + select { + case <-p.context.Done(): + return + default: + if !listen.Next(p.context) { + return + } + + for _, reservation := range listen.CurrentPage().Reservations { + for _, instance := range reservation.Instances { + p.recordGoodResult(instance) + } + } + } + + } +} + +func (p *fetchRequest) recordGoodResult(instance ec2.Instance) { + p.resultsLock.Lock() + defer p.resultsLock.Unlock() + + p.ec2Instances = append(p.ec2Instances, &ec2Instance{instance}) +} + +func (p *fetchRequest) recordErrResult(err error) { + p.resultsLock.Lock() + defer p.resultsLock.Unlock() + + p.errs = append(p.errs, err) + + p.cancel() +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/fetch_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/fetch_test.go new file mode 100644 index 00000000000..084322e06e8 --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/fetch_test.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" + "github.com/stretchr/testify/require" +) + +func Test_newAPIFetcher(t *testing.T) { + client := newMockEC2Client(0) + fetcher := newAPIFetcher([]ec2iface.ClientAPI{client}) + require.NotNil(t, fetcher) +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/mock_ec2_client_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/mock_ec2_client_test.go new file mode 100644 index 00000000000..66c9cf6c6c2 --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/mock_ec2_client_test.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" +) + +func newMockEC2Client(numResults int) mockEC2Client { + return mockEC2Client{numResults: numResults} +} + +type mockEC2Client struct { + ec2iface.ClientAPI + numResults int +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/mocks_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/mocks_test.go new file mode 100644 index 00000000000..542ddf4925e --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/mocks_test.go @@ -0,0 +1,85 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "context" + "sync" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" +) + +// mockFetcher is a fetcher that returns a customizable list of results, useful for testing. +type mockFetcher struct { + ec2Instances []*ec2Instance + err error + lock sync.Mutex +} + +func newMockFetcher(lbListeners []*ec2Instance, err error) *mockFetcher { + return &mockFetcher{ec2Instances: lbListeners, err: err} +} + +func (f *mockFetcher) fetch(ctx context.Context) ([]*ec2Instance, error) { + f.lock.Lock() + defer f.lock.Unlock() + + result := make([]*ec2Instance, len(f.ec2Instances)) + copy(result, f.ec2Instances) + + return result, f.err +} + +func (f *mockFetcher) setEC2s(newEC2s []*ec2Instance) { + f.lock.Lock() + defer f.lock.Unlock() + + f.ec2Instances = newEC2s +} + +func (f *mockFetcher) setError(err error) { + f.lock.Lock() + defer f.lock.Unlock() + + f.ec2Instances = []*ec2Instance{} + f.err = err +} + +func fakeEC2Instance() *ec2Instance { + runningCode := int64(16) + coreCount := int64(1) + threadsPerCore := int64(1) + publicDNSName := "ec2-1-2-3-4.us-west-1.compute.amazonaws.com" + publicIP := "1.2.3.4" + privateDNSName := "ip-5-6-7-8.us-west-1.compute.internal" + privateIP := "5.6.7.8" + instanceID := "i-123" + + instance := ec2.Instance{ + InstanceId: aws.String(instanceID), + InstanceType: ec2.InstanceTypeT2Medium, + Placement: &ec2.Placement{ + AvailabilityZone: aws.String("us-west-1a"), + }, + ImageId: aws.String("image-123"), + State: &ec2.InstanceState{ + Name: ec2.InstanceStateNameRunning, + Code: &runningCode, + }, + Monitoring: &ec2.Monitoring{ + State: ec2.MonitoringStateDisabled, + }, + CpuOptions: &ec2.CpuOptions{ + CoreCount: &coreCount, + ThreadsPerCore: &threadsPerCore, + }, + PublicDnsName: &publicDNSName, + PublicIpAddress: &publicIP, + PrivateDnsName: &privateDNSName, + PrivateIpAddress: &privateIP, + } + return &ec2Instance{ec2Instance: instance} +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go new file mode 100644 index 00000000000..4c9aa3e8b43 --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -0,0 +1,150 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" + "github.com/gofrs/uuid" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/autodiscover" + "github.com/elastic/beats/libbeat/autodiscover/template" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/bus" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" +) + +func init() { + autodiscover.Registry.AddProvider("aws_ec2", AutodiscoverBuilder) +} + +// Provider implements autodiscover provider for aws EC2s. +type Provider struct { + config *awsauto.Config + bus bus.Bus + templates *template.Mapper + startListener bus.Listener + stopListener bus.Listener + watcher *watcher + uuid uuid.UUID +} + +// AutodiscoverBuilder is the main builder for this provider. +func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodiscover.Provider, error) { + cfgwarn.Experimental("aws_ec2 autodiscover is experimental") + + config := awsauto.DefaultConfig() + err := c.Unpack(&config) + if err != nil { + return nil, err + } + + awsCfg, err := awscommon.GetAWSCredentials( + awscommon.ConfigAWS{ + AccessKeyID: config.AWSConfig.AccessKeyID, + SecretAccessKey: config.AWSConfig.SecretAccessKey, + SessionToken: config.AWSConfig.SessionToken, + ProfileName: config.AWSConfig.ProfileName, + }) + + // Construct MetricSet with a full regions list if there is no region specified. + if config.Regions == nil { + // set default region to make initial aws api call + awsCfg.Region = "us-west-1" + svcEC2 := ec2.New(awsCfg) + completeRegionsList, err := awsauto.GetRegions(svcEC2) + if err != nil { + return nil, err + } + + config.Regions = completeRegionsList + } + + var clients []ec2iface.ClientAPI + for _, region := range config.Regions { + if err != nil { + logp.Error(errors.Wrap(err, "error loading AWS config for aws_ec2 autodiscover provider")) + } + awsCfg.Region = region + clients = append(clients, ec2.New(awsCfg)) + } + + return internalBuilder(uuid, bus, config, newAPIFetcher(clients)) +} + +// internalBuilder is mainly intended for testing via mocks and stubs. +// it can be configured to use a fetcher that doesn't actually hit the AWS API. +func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetcher fetcher) (*Provider, error) { + mapper, err := template.NewConfigMapper(config.Templates) + if err != nil { + return nil, err + } + + p := &Provider{ + config: config, + bus: bus, + templates: &mapper, + uuid: uuid, + } + + p.watcher = newWatcher( + fetcher, + config.Period, + p.onWatcherStart, + p.onWatcherStop, + ) + + return p, nil +} + +// Start the autodiscover process. +func (p *Provider) Start() { + p.watcher.start() +} + +// Stop the autodiscover process. +func (p *Provider) Stop() { + p.watcher.stop() +} + +func (p *Provider) onWatcherStart(instanceID string, instance *ec2Instance) { + e := bus.Event{ + "start": true, + "provider": p.uuid, + "id": instanceID, + "aws": common.MapStr{ + "ec2": instance.toMap(), + }, + "cloud": instance.toCloudMap(), + "meta": common.MapStr{ + "aws": common.MapStr{ + "ec2": instance.toMap(), + }, + "cloud": instance.toCloudMap(), + }, + } + + if configs := p.templates.GetConfig(e); configs != nil { + e["config"] = configs + } + p.bus.Publish(e) +} + +func (p *Provider) onWatcherStop(instanceID string) { + e := bus.Event{ + "stop": true, + "id": instanceID, + "provider": p.uuid, + } + p.bus.Publish(e) +} + +func (p *Provider) String() string { + return "aws_ec2" +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go new file mode 100644 index 00000000000..f8216294006 --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go @@ -0,0 +1,110 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/bus" + "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" + "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws/test" +) + +func Test_internalBuilder(t *testing.T) { + instance := fakeEC2Instance() + instances := []*ec2Instance{instance} + fetcher := newMockFetcher(instances, nil) + log := logp.NewLogger("ec2") + pBus := bus.New(log, "test") + + cfg := &awsauto.Config{ + Regions: []string{"us-east-1a", "us-west-1b"}, + Period: time.Nanosecond, + } + + uuid, _ := uuid.NewV4() + provider, err := internalBuilder(uuid, pBus, cfg, fetcher) + require.NoError(t, err) + + startListener := pBus.Subscribe("start") + stopListener := pBus.Subscribe("stop") + listenerDone := make(chan struct{}) + defer close(listenerDone) + + var events test.TestEventAccumulator + go func() { + for { + select { + case e := <-startListener.Events(): + events.Add(e) + case e := <-stopListener.Events(): + events.Add(e) + case <-listenerDone: + return + } + } + }() + + // Let run twice to ensure that duplicates don't create two start events + // Since we're turning a list of assets into a list of changes the second once() call should be a noop + provider.watcher.once() + provider.watcher.once() + events.WaitForNumEvents(t, 1, time.Second) + + assert.Equal(t, 1, events.Len()) + + expectedStartEvent := bus.Event{ + "id": instance.instanceID(), + "provider": uuid, + "start": true, + "aws": common.MapStr{ + "ec2": instance.toMap(), + }, + "cloud": instance.toCloudMap(), + "meta": common.MapStr{ + "aws": common.MapStr{ + "ec2": instance.toMap(), + }, + "cloud": instance.toCloudMap(), + }, + } + + require.Equal(t, expectedStartEvent, events.Get()[0]) + + fetcher.setEC2s([]*ec2Instance{}) + + // Let run twice to ensure that duplicates don't cause an issue + provider.watcher.once() + provider.watcher.once() + events.WaitForNumEvents(t, 2, time.Second) + + require.Equal(t, 2, events.Len()) + + expectedStopEvent := bus.Event{ + "stop": true, + "id": awsauto.SafeString(instance.ec2Instance.InstanceId), + "provider": uuid, + } + + require.Equal(t, expectedStopEvent, events.Get()[1]) + + // Test that in an error situation nothing changes. + preErrorEventCount := events.Len() + fetcher.setError(errors.New("oops")) + + // Let run twice to ensure that duplicates don't cause an issue + provider.watcher.once() + provider.watcher.once() + + assert.Equal(t, preErrorEventCount, events.Len()) +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/watch.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/watch.go new file mode 100644 index 00000000000..4ae56465d9c --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/watch.go @@ -0,0 +1,107 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "context" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" +) + +type watcher struct { + // gen tracks changes we increment the 'generation' of each entry in the map. + gen uint64 + fetcher fetcher + onStart func(uuid string, lblMap *ec2Instance) + onStop func(uuid string) + done chan struct{} + ticker *time.Ticker + period time.Duration + ec2Instances map[string]uint64 + logger *logp.Logger +} + +func newWatcher( + fetcher fetcher, + period time.Duration, + onStart func(uuid string, instanceMap *ec2Instance), + onStop func(uuid string)) *watcher { + return &watcher{ + fetcher: fetcher, + onStart: onStart, + onStop: onStop, + done: make(chan struct{}), + ticker: time.NewTicker(period), + period: period, + ec2Instances: map[string]uint64{}, + logger: logp.NewLogger("autodiscover-ec2-watcher"), + } +} + +func (w *watcher) start() { + go w.forever() +} + +func (w *watcher) stop() { + close(w.done) +} + +func (w *watcher) forever() { + for { + select { + case <-w.done: + w.ticker.Stop() + return + case <-w.ticker.C: + err := w.once() + if err != nil { + logp.Error(errors.Wrap(err, "error while fetching AWS EC2s")) + } + } + } +} + +// once executes the watch loop a single time. +// This is mostly useful for testing. +func (w *watcher) once() error { + ctx, cancelCtx := context.WithTimeout(context.Background(), w.period) + defer cancelCtx() // Always cancel to avoid leak + + fetchedEC2s, err := w.fetcher.fetch(ctx) + if err != nil { + return err + } + w.logger.Debugf("fetched %d ec2 instances from AWS for autodiscover", len(fetchedEC2s)) + + oldGen := w.gen + w.gen++ + + // Increment the generation of all EC2s returned by the API request + for _, instance := range fetchedEC2s { + instanceID := awsauto.SafeString(instance.ec2Instance.InstanceId) + if _, exists := w.ec2Instances[instanceID]; !exists { + if w.onStart != nil { + w.onStart(instanceID, instance) + } + } + w.ec2Instances[instanceID] = w.gen + } + + // EC2s not seen in the API request get deleted + for uuid, entryGen := range w.ec2Instances { + if entryGen == oldGen { + if w.onStop != nil { + w.onStop(uuid) + delete(w.ec2Instances, uuid) + } + } + } + + return nil +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/watch_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/watch_test.go new file mode 100644 index 00000000000..83cae46efed --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/watch_test.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ec2 + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" +) + +func TestWatchTicks(t *testing.T) { + instances := []*ec2Instance{fakeEC2Instance()} + + lock := sync.Mutex{} + var startUUIDs []string + var startEC2s []*ec2Instance + var stopUUIDs []string + + fetcher := newMockFetcher(instances, nil) + watcher := newWatcher( + fetcher, + time.Millisecond, + func(uuid string, lbListener *ec2Instance) { + lock.Lock() + defer lock.Unlock() + + startUUIDs = append(startUUIDs, uuid) + startEC2s = append(startEC2s, lbListener) + }, + func(uuid string) { + lock.Lock() + defer lock.Unlock() + + stopUUIDs = append(stopUUIDs, uuid) + }) + defer watcher.stop() + + // Run through 10 ticks + for i := 0; i < 10; i++ { + err := watcher.once() + require.NoError(t, err) + } + + // The instanceID is the unique identifier used. + instanceIDs := []string{*instances[0].ec2Instance.InstanceId} + + // Test that we've seen one ec2 start, but none stop + assert.Equal(t, instanceIDs, startUUIDs) + assert.Len(t, stopUUIDs, 0) + assert.Equal(t, instances, startEC2s) + + // Stop the ec2 and test that we see a single stop + // and no change to starts + fetcher.setEC2s(nil) + watcher.once() + + assert.Equal(t, instanceIDs, startUUIDs) + assert.Equal(t, instanceIDs, stopUUIDs) +} diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/fetch.go b/x-pack/libbeat/autodiscover/providers/aws/elb/fetch.go index 1cdb8e89984..33a7e8bb56a 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/fetch.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/fetch.go @@ -15,8 +15,6 @@ import ( "github.com/elastic/beats/libbeat/logp" ) -const logSelector = "autodiscover-elb-fetch" - // fetcher is an interface that can fetch a list of lbListener (load balancer + listener) objects without pagination being necessary. type fetcher interface { fetch(ctx context.Context) ([]*lbListener, error) @@ -64,7 +62,7 @@ type apiFetcher struct { client elasticloadbalancingv2iface.ClientAPI } -func newAPIFetcher(ctx context.Context, clients []elasticloadbalancingv2iface.ClientAPI) fetcher { +func newAPIFetcher(clients []elasticloadbalancingv2iface.ClientAPI) fetcher { fetchers := make([]fetcher, len(clients)) for idx, client := range clients { fetchers[idx] = &apiFetcher{client} @@ -89,6 +87,7 @@ func (f *apiFetcher) fetch(ctx context.Context) ([]*lbListener, error) { taskPool: sync.Pool{}, context: ctx, cancel: cancel, + logger: logp.NewLogger("autodiscover-elb-fetch"), } // Limit concurrency against the AWS API by creating a pool of objects @@ -112,6 +111,7 @@ type fetchRequest struct { pendingTasks sync.WaitGroup context context.Context cancel func() + logger *logp.Logger } func (p *fetchRequest) fetch() ([]*lbListener, error) { @@ -138,14 +138,14 @@ func (p *fetchRequest) fetchAllPages() { for { select { case <-p.context.Done(): - logp.Debug(logSelector, "done fetching ELB pages, context cancelled") + p.logger.Debug("done fetching ELB pages, context cancelled") return default: if !p.fetchNextPage() { - logp.Debug(logSelector, "fetched all ELB pages") + p.logger.Debug("fetched all ELB pages") return } - logp.Debug(logSelector, "fetched ELB page") + p.logger.Debug("fetched ELB page") } } } diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/fetch_test.go b/x-pack/libbeat/autodiscover/providers/aws/elb/fetch_test.go index 747c9738ed3..c1eadd70a3a 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/fetch_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/fetch_test.go @@ -5,7 +5,6 @@ package elb import ( - "context" "testing" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface" @@ -14,6 +13,6 @@ import ( func Test_newAPIFetcher(t *testing.T) { client := newMockELBClient(0) - fetcher := newAPIFetcher(context.TODO(), []elasticloadbalancingv2iface.ClientAPI{client}) + fetcher := newAPIFetcher([]elasticloadbalancingv2iface.ClientAPI{client}) require.NotNil(t, fetcher) } diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/lblistener.go b/x-pack/libbeat/autodiscover/providers/aws/elb/lblistener.go index 94e49867781..b673113872d 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/lblistener.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/lblistener.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/elastic/beats/libbeat/common" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" ) // lbListener is a tuple type representing an elasticloadbalancingv2.Listener and its associated elasticloadbalancingv2.LoadBalancer. @@ -21,8 +22,8 @@ func (l *lbListener) toMap() common.MapStr { // We fully spell out listener_arn to avoid confusion with the ARN for the whole ELB m := common.MapStr{ "listener_arn": l.listener.ListenerArn, - "load_balancer_arn": safeStrp(l.lb.LoadBalancerArn), - "host": safeStrp(l.lb.DNSName), + "load_balancer_arn": awsauto.SafeString(l.lb.LoadBalancerArn), + "host": awsauto.SafeString(l.lb.DNSName), "protocol": l.listener.Protocol, "type": string(l.lb.Type), "scheme": l.lb.Scheme, @@ -31,7 +32,7 @@ func (l *lbListener) toMap() common.MapStr { "state": l.stateMap(), "ip_address_type": string(l.lb.IpAddressType), "security_groups": l.lb.SecurityGroups, - "vpc_id": safeStrp(l.lb.VpcId), + "vpc_id": awsauto.SafeString(l.lb.VpcId), "ssl_policy": l.listener.SslPolicy, } @@ -42,18 +43,6 @@ func (l *lbListener) toMap() common.MapStr { return m } -// safeStrp makes handling AWS *string types easier. -// The AWS lib never returns plain strings, always using pointers, probably for memory efficiency reasons. -// This is a bit odd, because strings are just pointers into byte arrays, however this is the choice they've made. -// This will return the plain version of the given string or an empty string if the pointer is null -func safeStrp(strp *string) string { - if strp == nil { - return "" - } - - return *strp -} - func (l *lbListener) toCloudMap() common.MapStr { m := common.MapStr{} diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index bad7fae4b15..b965f9ee1e2 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -5,10 +5,7 @@ package elb import ( - "context" - - awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" - + "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface" "github.com/gofrs/uuid" @@ -19,6 +16,8 @@ import ( "github.com/elastic/beats/libbeat/common/bus" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" ) func init() { @@ -27,7 +26,7 @@ func init() { // Provider implements autodiscover provider for aws ELBs. type Provider struct { - config *Config + config *awsauto.Config bus bus.Bus builders autodiscover.Builders appenders autodiscover.Appenders @@ -42,12 +41,32 @@ type Provider struct { func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodiscover.Provider, error) { cfgwarn.Experimental("aws_elb autodiscover is experimental") - config := defaultConfig() + config := awsauto.DefaultConfig() err := c.Unpack(&config) if err != nil { return nil, err } + awsCfg, err := awscommon.GetAWSCredentials(awscommon.ConfigAWS{ + AccessKeyID: config.AWSConfig.AccessKeyID, + SecretAccessKey: config.AWSConfig.SecretAccessKey, + SessionToken: config.AWSConfig.SessionToken, + ProfileName: config.AWSConfig.ProfileName, + }) + + // Construct MetricSet with a full regions list if there is no region specified. + if config.Regions == nil { + // set default region to make initial aws api call + awsCfg.Region = "us-west-1" + svcEC2 := ec2.New(awsCfg) + completeRegionsList, err := awsauto.GetRegions(svcEC2) + if err != nil { + return nil, err + } + + config.Regions = completeRegionsList + } + var clients []elasticloadbalancingv2iface.ClientAPI for _, region := range config.Regions { awsCfg, err := awscommon.GetAWSCredentials(awscommon.ConfigAWS{ @@ -63,32 +82,20 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis clients = append(clients, elasticloadbalancingv2.New(awsCfg)) } - return internalBuilder(uuid, bus, config, newAPIFetcher(context.TODO(), clients)) + return internalBuilder(uuid, bus, config, newAPIFetcher(clients)) } // internalBuilder is mainly intended for testing via mocks and stubs. // it can be configured to use a fetcher that doesn't actually hit the AWS API. -func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *Config, fetcher fetcher) (*Provider, error) { +func internalBuilder(uuid uuid.UUID, bus bus.Bus, config *awsauto.Config, fetcher fetcher) (*Provider, error) { mapper, err := template.NewConfigMapper(config.Templates) if err != nil { return nil, err } - builders, err := autodiscover.NewBuilders(config.Builders, nil) - if err != nil { - return nil, err - } - - appenders, err := autodiscover.NewAppenders(config.Appenders) - if err != nil { - return nil, err - } - p := &Provider{ config: config, bus: bus, - builders: builders, - appenders: appenders, templates: &mapper, uuid: uuid, } diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go index 0378bb7f93e..0f7bd0f66c6 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/bus" "github.com/elastic/beats/libbeat/logp" + awsauto "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws" ) type testEventAccumulator struct { @@ -67,7 +68,7 @@ func Test_internalBuilder(t *testing.T) { fetcher := newMockFetcher(lbls, nil) pBus := bus.New(log, "test") - cfg := &Config{ + cfg := &awsauto.Config{ Regions: []string{"us-east-1a", "us-west-1b"}, Period: time.Nanosecond, } diff --git a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go new file mode 100644 index 00000000000..353b6d62631 --- /dev/null +++ b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package test + +import ( + "sync" + "testing" + "time" + + "github.com/elastic/beats/libbeat/common/bus" +) + +// TestEventAccumulator defined a list of events for testing +type TestEventAccumulator struct { + events []bus.Event + lock sync.Mutex +} + +// Add expends events +func (tea *TestEventAccumulator) Add(e bus.Event) { + tea.lock.Lock() + defer tea.lock.Unlock() + + tea.events = append(tea.events, e) +} + +// Len returns length of events +func (tea *TestEventAccumulator) Len() int { + tea.lock.Lock() + defer tea.lock.Unlock() + + return len(tea.events) +} + +// Get copies the event and return it +func (tea *TestEventAccumulator) Get() []bus.Event { + tea.lock.Lock() + defer tea.lock.Unlock() + + res := make([]bus.Event, len(tea.events)) + copy(res, tea.events) + return res +} + +// WaitForNumEvents waits to get target length of events +func (tea *TestEventAccumulator) WaitForNumEvents(t *testing.T, targetLen int, timeout time.Duration) { + start := time.Now() + + for time.Now().Sub(start) < timeout { + if tea.Len() >= targetLen { + return + } + time.Sleep(time.Millisecond) + } + + t.Fatalf("Timed out waiting for num events to be %d", targetLen) +} diff --git a/x-pack/libbeat/cmd/inject.go b/x-pack/libbeat/cmd/inject.go index 71d72a6c7c2..658209bb99b 100644 --- a/x-pack/libbeat/cmd/inject.go +++ b/x-pack/libbeat/cmd/inject.go @@ -12,6 +12,7 @@ import ( _ "github.com/elastic/beats/x-pack/libbeat/management" // register autodiscover providers + _ "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws/ec2" _ "github.com/elastic/beats/x-pack/libbeat/autodiscover/providers/aws/elb" ) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index b010788aa1b..28ce31cc400 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -659,6 +659,7 @@ metricbeat.modules: - state_service - state_persistentvolume - state_persistentvolumeclaim + - state_storageclass period: 10s hosts: ["kube-state-metrics:8080"] @@ -901,7 +902,7 @@ metricbeat.modules: #password: "secret" # This can be used for service account based authorization: - # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt @@ -1432,6 +1433,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -1525,6 +1532,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting diff --git a/x-pack/metricbeat/module/tomcat/_meta/docs.asciidoc b/x-pack/metricbeat/module/tomcat/_meta/docs.asciidoc index cf7c85845e3..11966657cb8 100644 --- a/x-pack/metricbeat/module/tomcat/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/tomcat/_meta/docs.asciidoc @@ -4,6 +4,12 @@ This module periodically fetches JMX metrics from Apache Tomcat. === Compatibility The module has been tested with Tomcat 7.0.24 and 9.0.24. Other versions are expected to work. +[float] +== Dashboard +An overview dashboard for Kibana is already included: + +image::./images/metricbeat-tomcat-overview.png[] + [float] === Usage The Tomcat module requires <>to fetch JMX metrics. Refer to the link for instructions about how to use Jolokia. diff --git a/x-pack/metricbeat/module/tomcat/_meta/kibana/7/dashboard/Metricbeat-Tomcat-overview.json b/x-pack/metricbeat/module/tomcat/_meta/kibana/7/dashboard/Metricbeat-Tomcat-overview.json new file mode 100644 index 00000000000..195256aad93 --- /dev/null +++ b/x-pack/metricbeat/module/tomcat/_meta/kibana/7/dashboard/Metricbeat-Tomcat-overview.json @@ -0,0 +1,1026 @@ +{ + "objects": [ + { + "attributes": { + "description": "An overview dashboards of all key metrics included in the Tomcat Metricbeat module", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "1", + "w": 18, + "x": 16, + "y": 18 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "title": "Other Memory usage", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "2", + "w": 25, + "x": 0, + "y": 0 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "title": "Heap memory usage", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "3", + "w": 16, + "x": 0, + "y": 18 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "title": "Total Threads", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "5", + "w": 14, + "x": 34, + "y": 18 + }, + "panelIndex": "5", + "panelRefName": "panel_3", + "title": "Cache Size", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "6", + "w": 23, + "x": 25, + "y": 0 + }, + "panelIndex": "6", + "panelRefName": "panel_4", + "title": "Total requests", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "7", + "w": 16, + "x": 0, + "y": 9 + }, + "panelIndex": "7", + "panelRefName": "panel_5", + "title": "Bytes sent/received", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "8", + "w": 18, + "x": 16, + "y": 9 + }, + "panelIndex": "8", + "panelRefName": "panel_6", + "title": "Processing time ms", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 9, + "i": "9", + "w": 14, + "x": 34, + "y": 9 + }, + "panelIndex": "9", + "panelRefName": "panel_7", + "title": "Cache Hit Total", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Metricbeat Tomcat] Overview", + "version": 1 + }, + "id": "b7675d90-ec34-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "references": [ + { + "id": "618130c0-ec31-11e9-a4bb-7b5324058fcc", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "e037f620-ec30-11e9-a4bb-7b5324058fcc", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "17ec8fd0-ec32-11e9-a4bb-7b5324058fcc", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "b3e833d0-ec32-11e9-a4bb-7b5324058fcc", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "9d0616e0-ec33-11e9-a4bb-7b5324058fcc", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "7bcc3ae0-ec33-11e9-a4bb-7b5324058fcc", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "9343a810-ec34-11e9-a4bb-7b5324058fcc", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "38f55440-ec33-11e9-a4bb-7b5324058fcc", + "name": "panel_7", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2020-02-06T17:15:24.232Z", + "version": "WzEzMDUsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Other Memory usage [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "legend_position": "right", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(159,5,0,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Used", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.other.usage.used", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "bytes", + "id": "f34791d0-ec30-11e9-a38d-abb22848ef7c", + "label": "Committed", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.other.usage.committed", + "id": "f34791d1-ec30-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(115,216,255,1)", + "fill": 0.5, + "formatter": "bytes", + "hidden": false, + "id": "2acecf60-ec31-11e9-a38d-abb22848ef7c", + "label": "Init", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.other.usage.init", + "id": "2acecf61-ec31-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(226,115,0,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "36d48200-ec31-11e9-a38d-abb22848ef7c", + "label": "Max", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.other.usage.max", + "id": "36d48201-ec31-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Other Memory usage [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "618130c0-ec31-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:15:24.232Z", + "version": "WzEzMDYsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Heap memory usage [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(159,5,0,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "9057a880-ec30-11e9-a38d-abb22848ef7c", + "label": "Used", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.heap.usage.used", + "id": "9057a881-ec30-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "bytes", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Committed", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.heap.usage.committed", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(115,216,255,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "78d8a4c0-ec30-11e9-a38d-abb22848ef7c", + "label": "Init", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.heap.usage.init", + "id": "78d8cbd0-ec30-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(254,146,0,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "855cd630-ec30-11e9-a38d-abb22848ef7c", + "label": "Max", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.memory.heap.usage.max", + "id": "855cd631-ec30-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Heap memory usage [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "e037f620-ec30-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:15:24.232Z", + "version": "WzEzMDcsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total Threads [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "legend_position": "bottom", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "hide_in_legend": 0, + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Total", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.threading.total", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "steps": 0, + "terms_field": "tomcat.threading.mbean" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": 0.5, + "formatter": "number", + "id": "2e15eea0-ec32-11e9-a38d-abb22848ef7c", + "label": "Peak", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.threading.peak", + "id": "2e15eea1-ec32-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Total Threads [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "17ec8fd0-ec32-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:15:24.232Z", + "version": "WzEzMDgsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Cache Size [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "legend_position": "bottom", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "bytes", + "id": "814df2c0-ec32-11e9-a38d-abb22848ef7c", + "label": "Total", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.cache.size.total.kb", + "id": "814df2c1-ec32-11e9-a38d-abb22848ef7c", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(254,146,0,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Max", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.cache.size.max.kb", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value/1024}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Cache Size [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "b3e833d0-ec32-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:15:24.232Z", + "version": "WzEzMDksNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total requests [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "legend_position": "bottom", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "hidden": false, + "id": "a2ae7420-ec33-11e9-a38d-abb22848ef7c", + "label": "Total", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.requests.total", + "id": "a2ae7421-ec33-11e9-a38d-abb22848ef7c", + "mode": "raw", + "sigma": "1", + "type": "avg" + }, + { + "field": "a2ae7421-ec33-11e9-a38d-abb22848ef7c", + "id": "40d82510-ee8c-11e9-a452-e36b1c5ca82b", + "type": "derivative", + "unit": "" + }, + { + "field": "40d82510-ee8c-11e9-a452-e36b1c5ca82b", + "id": "a3934920-ee94-11e9-a452-e36b1c5ca82b", + "type": "positive_only", + "unit": "" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Errors", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.requests.errors.total", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + }, + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "be872850-ee94-11e9-a452-e36b1c5ca82b", + "type": "derivative", + "unit": "" + }, + { + "field": "be872850-ee94-11e9-a452-e36b1c5ca82b", + "id": "c2290730-ee94-11e9-a452-e36b1c5ca82b", + "type": "positive_only", + "unit": "" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Total requests [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "9d0616e0-ec33-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:46:49.245Z", + "version": "WzE4MzAsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Bytes sent/received [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "legend_position": "bottom", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Received", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.requests.bytes.sent", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + }, + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "a0d8b6a0-ee8c-11e9-a452-e36b1c5ca82b", + "type": "derivative", + "unit": "" + }, + { + "field": "a0d8b6a0-ee8c-11e9-a452-e36b1c5ca82b", + "id": "e3099fa0-ee94-11e9-a452-e36b1c5ca82b", + "type": "positive_only", + "unit": "" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(115,216,255,1)", + "fill": 0.5, + "formatter": "number", + "id": "5310fbe0-ec33-11e9-a38d-abb22848ef7c", + "label": "Sent", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.requests.bytes.received", + "id": "5310fbe1-ec33-11e9-a38d-abb22848ef7c", + "type": "avg" + }, + { + "field": "5310fbe1-ec33-11e9-a38d-abb22848ef7c", + "id": "bbfe0610-ee8c-11e9-a452-e36b1c5ca82b", + "type": "derivative", + "unit": "" + }, + { + "field": "bbfe0610-ee8c-11e9-a452-e36b1c5ca82b", + "id": "e6da0890-ee94-11e9-a452-e36b1c5ca82b", + "type": "positive_only", + "unit": "" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Bytes sent/received [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "7bcc3ae0-ec33-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:46:17.902Z", + "version": "WzE4MjEsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Processing time ms [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "ms,ms,3", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.requests.processing.ms", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + }, + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "cbb9e2e0-ee8c-11e9-a452-e36b1c5ca82b", + "type": "derivative", + "unit": "" + }, + { + "field": "cbb9e2e0-ee8c-11e9-a452-e36b1c5ca82b", + "id": "d0327910-ee94-11e9-a452-e36b1c5ca82b", + "type": "positive_only", + "unit": "" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 0, + "time_field": "", + "type": "timeseries" + }, + "title": "Processing time ms [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "9343a810-ec34-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:45:36.872Z", + "version": "WzE4MTQsNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Cache Hit Total [Metricbeat Tomcat]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Total", + "line_width": 1, + "metrics": [ + { + "field": "tomcat.cache.hit.total", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 0, + "time_field": "", + "type": "timeseries" + }, + "title": "Cache Hit Total [Metricbeat Tomcat]", + "type": "metrics" + } + }, + "id": "38f55440-ec33-11e9-a4bb-7b5324058fcc", + "migrationVersion": { + "visualization": "7.4.2" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-02-06T17:15:24.232Z", + "version": "WzEzMTMsNV0=" + } + ], + "version": "8.0.0-SNAPSHOT" +} diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 1745ebdb730..bcdbf155a04 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -437,6 +437,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + #----------------------------- Logstash output --------------------------------- #output.logstash: # Boolean flag to enable or disable the output module. @@ -530,6 +536,12 @@ output.elasticsearch: # never, once, and freely. Default is never. #ssl.renegotiation: never + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256 + # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting