diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 59eea0a8101..e3ce15e3500 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -74,9 +74,12 @@ CHANGELOG* /x-pack/elastic-agent/ @elastic/elastic-agent-control-plane /x-pack/filebeat @elastic/elastic-agent-data-plane /x-pack/filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. +/x-pack/filebeat/input/awscloudwatch/ @elastic/obs-cloud-monitoring +/x-pack/filebeat/input/awss3/ @elastic/obs-cloud-monitoring /x-pack/filebeat/input/gcppubsub/ @elastic/security-external-integrations /x-pack/filebeat/input/http_endpoint/ @elastic/security-external-integrations /x-pack/filebeat/input/httpjson/ @elastic/security-external-integrations +/x-pack/filebeat/input/lumberjack/ @elastic/security-external-integrations /x-pack/filebeat/input/netflow/ @elastic/security-external-integrations /x-pack/filebeat/input/o365audit/ @elastic/security-external-integrations /x-pack/filebeat/module/ @elastic/integrations diff --git a/.github/workflows/check-audtibeat.yml b/.github/workflows/check-audtibeat.yml new file mode 100644 index 00000000000..20e8459d557 --- /dev/null +++ b/.github/workflows/check-audtibeat.yml @@ -0,0 +1,27 @@ +name: check-auditbeat + +on: + pull_request: + paths: + - '.github/workflows/check-auditbeat.yml' + - 'auditbeat/**' + - 'x-pack/auditbeat/**' + +env: + BEAT_MODULE: 'auditbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml new file mode 100644 index 00000000000..304b5382c4a --- /dev/null +++ b/.github/workflows/check-default.yml @@ -0,0 +1,28 @@ +name: check-default + +on: + pull_request: + push: + branches: + - main + - 7.1* + - 8.* + +permissions: + contents: read + pull-requests: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check-default + run: | + go install github.com/magefile/mage + make check-default diff --git a/.github/workflows/check-dev-tools.yml b/.github/workflows/check-dev-tools.yml new file mode 100644 index 00000000000..df71c821a10 --- /dev/null +++ b/.github/workflows/check-dev-tools.yml @@ -0,0 +1,26 @@ +name: check-dev-tools + +on: + pull_request: + paths: + - '.github/workflows/check-dev-tools.yml' + - 'dev-tools/**' + - 'libbeat/scripts/Makefile' + +env: + BEAT_MODULE: 'dev-tools' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml new file mode 100644 index 00000000000..4d56b80fa31 --- /dev/null +++ b/.github/workflows/check-docs.yml @@ -0,0 +1,34 @@ +name: check-docs + +on: + pull_request: + paths: + - '.github/workflows/check-docs.yml' + - '**/*.asciidoc' + - '**/*.md' + - 'deploy/kubernetes/*-kubernetes.yaml' + +permissions: + contents: read + pull-requests: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install libpcap-dev + run: sudo apt-get install -y libpcap-dev + - name: Install libsystemd-dev + run: sudo apt-get install -y libsystemd-dev + - name: Install librpm-dev + run: sudo apt-get install -y librpm-dev + - name: Run check + run: | + go install github.com/magefile/mage + make check diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml new file mode 100644 index 00000000000..6eca3a3d97c --- /dev/null +++ b/.github/workflows/check-filebeat.yml @@ -0,0 +1,29 @@ +name: check-filebeat + +on: + pull_request: + paths: + - '.github/workflows/check-filebeat.yml' + - 'filebeat/**' + - 'x-pack/filebeat/**' + +env: + BEAT_MODULE: 'filebeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install libsystemd-dev + run: sudo apt-get install -y libsystemd-dev + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-heartbeat.yml b/.github/workflows/check-heartbeat.yml new file mode 100644 index 00000000000..a0242812a52 --- /dev/null +++ b/.github/workflows/check-heartbeat.yml @@ -0,0 +1,27 @@ +name: check-heartbeat + +on: + pull_request: + paths: + - '.github/workflows/check-heartbeat.yml' + - 'heartbeat/**' + - 'x-pack/heartbeat/**' + +env: + BEAT_MODULE: 'heartbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml new file mode 100644 index 00000000000..ec76fbefbc9 --- /dev/null +++ b/.github/workflows/check-libbeat.yml @@ -0,0 +1,29 @@ +name: check-libbeat + +on: + pull_request: + paths: + - '.github/workflows/check-packetbeat.yml' + - 'libbeat/**' + - 'x-pack/libbeat/**' + +env: + BEAT_MODULE: 'libbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install libpcap-dev + run: sudo apt-get install -y libpcap-dev + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml new file mode 100644 index 00000000000..54f4753531d --- /dev/null +++ b/.github/workflows/check-metricbeat.yml @@ -0,0 +1,27 @@ +name: check-metricbeat + +on: + pull_request: + paths: + - '.github/workflows/check-metricbeat.yml' + - 'metricbeat/**' + - 'x-pack/metricbeat/**' + +env: + BEAT_MODULE: 'metricbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml new file mode 100644 index 00000000000..847e13e503f --- /dev/null +++ b/.github/workflows/check-packetbeat.yml @@ -0,0 +1,29 @@ +name: check-packetbeat + +on: + pull_request: + paths: + - '.github/workflows/check-packetbeat.yml' + - 'packetbeat/**' + - 'x-pack/packetbeat/**' + +env: + BEAT_MODULE: 'packetbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install libpcap-dev + run: sudo apt-get install -y libpcap-dev + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-winlogbeat.yml b/.github/workflows/check-winlogbeat.yml new file mode 100644 index 00000000000..41d7b95c0dd --- /dev/null +++ b/.github/workflows/check-winlogbeat.yml @@ -0,0 +1,27 @@ +name: check-winlogbeat + +on: + pull_request: + paths: + - '.github/workflows/check-winlogbeat.yml' + - 'winlogbeat/**' + - 'x-pack/winlogbeat/**' + +env: + BEAT_MODULE: 'winlogbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-auditbeat.yml b/.github/workflows/check-xpack-auditbeat.yml new file mode 100644 index 00000000000..1902de3942e --- /dev/null +++ b/.github/workflows/check-xpack-auditbeat.yml @@ -0,0 +1,29 @@ +name: check-x-pack-auditbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-auditbeat.yml' + - 'x-pack/auditbeat/**' + - 'auditbeat/**' + +env: + BEAT_MODULE: 'x-pack/auditbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install librpm-dev + run: sudo apt-get install -y librpm-dev + - name: Run check/update + uses: magefile/mage-action@v2 + with: + args: check update + workdir: "${{ env.BEAT_MODULE }}" diff --git a/.github/workflows/check-xpack-filebeat.yml b/.github/workflows/check-xpack-filebeat.yml new file mode 100644 index 00000000000..cd63315b0cd --- /dev/null +++ b/.github/workflows/check-xpack-filebeat.yml @@ -0,0 +1,29 @@ +name: check-x-pack-filebeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-filebeat.yml' + - 'x-pack/filebeat/**' + - 'filebeat/**' + +env: + BEAT_MODULE: 'x-pack/filebeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install libpcap-dev + run: sudo apt-get install -y libpcap-dev + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-functionbeat.yml b/.github/workflows/check-xpack-functionbeat.yml new file mode 100644 index 00000000000..5ce27961a21 --- /dev/null +++ b/.github/workflows/check-xpack-functionbeat.yml @@ -0,0 +1,27 @@ +name: check-x-pack-functionbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-functionbeat.yml' + - 'x-pack/functionbeat/**' + - 'functionbeat/**' + +env: + BEAT_MODULE: 'x-pack/functionbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-heartbeat.yml b/.github/workflows/check-xpack-heartbeat.yml new file mode 100644 index 00000000000..8a3eceb499e --- /dev/null +++ b/.github/workflows/check-xpack-heartbeat.yml @@ -0,0 +1,27 @@ +name: check-x-pack-heartbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-heartbeat.yml' + - 'x-pack/heartbeat/**' + - 'heartbeat/**' + +env: + BEAT_MODULE: 'x-pack/heartbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-libbeat.yml b/.github/workflows/check-xpack-libbeat.yml new file mode 100644 index 00000000000..4428b8ad900 --- /dev/null +++ b/.github/workflows/check-xpack-libbeat.yml @@ -0,0 +1,27 @@ +name: check-x-pack-libbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-libbeat.yml' + - 'x-pack/libbeat/**' + - 'libbeat/**' + +env: + BEAT_MODULE: 'x-pack/libbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-metricbeat.yml b/.github/workflows/check-xpack-metricbeat.yml new file mode 100644 index 00000000000..e3bd576c386 --- /dev/null +++ b/.github/workflows/check-xpack-metricbeat.yml @@ -0,0 +1,27 @@ +name: check-x-pack-metricbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-metricbeat.yml' + - 'x-pack/metricbeat/**' + - 'metricbeat/**' + +env: + BEAT_MODULE: 'x-pack/metricbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-osquerybeat.yml b/.github/workflows/check-xpack-osquerybeat.yml new file mode 100644 index 00000000000..b97ec590526 --- /dev/null +++ b/.github/workflows/check-xpack-osquerybeat.yml @@ -0,0 +1,27 @@ +name: check-x-pack-osquerybeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-osquerybeat.yml' + - 'x-pack/osquerybeat/**' + - 'osquerybeat/**' + +env: + BEAT_MODULE: 'x-pack/osquerybeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} check update + make check-no-changes diff --git a/.github/workflows/check-xpack-packetbeat.yml b/.github/workflows/check-xpack-packetbeat.yml new file mode 100644 index 00000000000..5469bdaf80e --- /dev/null +++ b/.github/workflows/check-xpack-packetbeat.yml @@ -0,0 +1,29 @@ +name: check-x-pack-packetbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-packetbeat.yml' + - 'x-pack/packetbeat/**' + - 'packetbeat/**' + +env: + BEAT_MODULE: 'x-pack/packetbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install libpcap-dev + run: sudo apt-get install -y libpcap-dev + - name: Run check/update + uses: magefile/mage-action@v2 + with: + args: check update + workdir: "${{ env.BEAT_MODULE }}" diff --git a/.github/workflows/check-xpack-winlogbeat.yml b/.github/workflows/check-xpack-winlogbeat.yml new file mode 100644 index 00000000000..3a18353df6b --- /dev/null +++ b/.github/workflows/check-xpack-winlogbeat.yml @@ -0,0 +1,27 @@ +name: check-x-pack-winlogbeat + +on: + pull_request: + paths: + - '.github/workflows/check-xpack-winlogbeat.yml' + - 'x-pack/winlogbeat/**' + - 'winlogbeat/**' + +env: + BEAT_MODULE: 'x-pack/winlogbeat' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Fetch Go version from .go-version + run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run check/update + uses: magefile/mage-action@v2 + with: + args: check update + workdir: "${{ env.BEAT_MODULE }}" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 6ba46de8a2f..981aceb49e6 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -26,24 +26,24 @@ jobs: GOOS: ${{ matrix.GOOS }} run: echo Go GOOS=$GOOS - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # Uses Go version from the repository. - name: Read .go-version file id: goversion run: echo "::set-output name=version::$(cat .go-version)" - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: "${{ steps.goversion.outputs.version }}" - name: golangci-lint env: GOOS: ${{ matrix.GOOS }} - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.44.2 + version: v1.47.2 # Give the job more time to execute. # Regarding `--whole-files`, the linter is supposed to support linting of changed a patch only but, diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml index 4ceb41bdcfb..3d01b6bc4e5 100644 --- a/.github/workflows/macos-auditbeat.yml +++ b/.github/workflows/macos-auditbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 6f392dff9e4..55c2d50fa3f 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -42,12 +42,12 @@ jobs: echo Changed GOPATH to ${{ env.MACOS_GOPATH }} echo "GOPATH="${{ env.MACOS_GOPATH }} >> $GITHUB_ENV - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ github.event.inputs.go_version }} - name: Install dependencies - run: go get -u github.com/magefile/mage - - uses: actions/checkout@v2 + run: go install github.com/magefile/mage + - uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{ env.GITHUB_REF }} diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml index 3939a53b48c..2c5d6b8c181 100644 --- a/.github/workflows/macos-filebeat.yml +++ b/.github/workflows/macos-filebeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-heartbeat.yml b/.github/workflows/macos-heartbeat.yml index 85fddbc5886..a3d612537a2 100644 --- a/.github/workflows/macos-heartbeat.yml +++ b/.github/workflows/macos-heartbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-metricbeat.yml b/.github/workflows/macos-metricbeat.yml index 19a05982154..56c3100ac5c 100644 --- a/.github/workflows/macos-metricbeat.yml +++ b/.github/workflows/macos-metricbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-packetbeat.yml b/.github/workflows/macos-packetbeat.yml index c4dbb7f0449..d9e89368543 100644 --- a/.github/workflows/macos-packetbeat.yml +++ b/.github/workflows/macos-packetbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-auditbeat.yml b/.github/workflows/macos-xpack-auditbeat.yml index f03a4d83945..5da201dfa61 100644 --- a/.github/workflows/macos-xpack-auditbeat.yml +++ b/.github/workflows/macos-xpack-auditbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-filebeat.yml b/.github/workflows/macos-xpack-filebeat.yml index 236d0ad6337..ae7c0c4f35d 100644 --- a/.github/workflows/macos-xpack-filebeat.yml +++ b/.github/workflows/macos-xpack-filebeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-functionbeat.yml b/.github/workflows/macos-xpack-functionbeat.yml index 0c39a2c842b..d20c714a489 100644 --- a/.github/workflows/macos-xpack-functionbeat.yml +++ b/.github/workflows/macos-xpack-functionbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml index 71a3b1da2f7..fcbc5e9b476 100644 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ b/.github/workflows/macos-xpack-heartbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-metricbeat.yml b/.github/workflows/macos-xpack-metricbeat.yml index ca2d4a30b14..330bd1e48e4 100644 --- a/.github/workflows/macos-xpack-metricbeat.yml +++ b/.github/workflows/macos-xpack-metricbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-osquerybeat.yml b/.github/workflows/macos-xpack-osquerybeat.yml index cbc7f08a3fe..af7a58dd8ab 100644 --- a/.github/workflows/macos-xpack-osquerybeat.yml +++ b/.github/workflows/macos-xpack-osquerybeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/macos-xpack-packetbeat.yml b/.github/workflows/macos-xpack-packetbeat.yml index 25f3508365d..163f6a9aaa8 100644 --- a/.github/workflows/macos-xpack-packetbeat.yml +++ b/.github/workflows/macos-xpack-packetbeat.yml @@ -20,11 +20,14 @@ jobs: - uses: actions/checkout@v2 - name: Fetch Go version from .go-version run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} + - uses: actions/setup-python@v4 + with: + python-version: '3.9' - name: Install dependencies - run: go get -u github.com/magefile/mage + run: go install github.com/magefile/mage - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml index d7e54e0a632..029e80f3b4b 100644 --- a/.github/workflows/opentelemetry.yml +++ b/.github/workflows/opentelemetry.yml @@ -3,6 +3,23 @@ name: OpenTelemetry Export Trace on: workflow_run: workflows: + - check-auditbeat + - check-dev-tools + - check-filebeat + - check-heartbeat + - check-libbeat + - check-metricbeat + - check-packetbeat + - check-winlogbeat + - check-x-pack-auditbeat + - check-x-pack-filebeat + - check-x-pack-functionbeat + - check-x-pack-heartbeat + - check-x-pack-libbeat + - check-x-pack-metricbeat + - check-x-pack-osquerybeat + - check-x-pack-packetbeat + - check-x-pack-winlogbeat - golangci-lint - auditbeat - filebeat @@ -16,6 +33,8 @@ on: - x-pack-metricbeat - x-pack-osquerybeat - x-pack-packetbeat + - check-default + - check-docs types: [completed] jobs: diff --git a/.go-version b/.go-version index 4512502b629..8e8b0a9335a 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.17.12 +1.18.5 diff --git a/.golangci.yml b/.golangci.yml index 3246ea7c8c3..763ed2bc2e9 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,6 +1,3 @@ -# DO NOT EDIT! -# This file is a rendered template, the source can be found in "./dev-tools/templates/.golangci.yml" -# # options for analysis running run: # timeout for analysis, e.g. 30s, 5m, default is 1m @@ -105,7 +102,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.17.12" + go: "1.18.5" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -125,19 +122,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.17.12" + go: "1.18.5" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.17.12" + go: "1.18.5" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.17.12" + go: "1.18.5" gosec: excludes: diff --git a/.mergify.yml b/.mergify.yml index 5edb010bc61..1dda587aa05 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -129,6 +129,24 @@ pull_request_rules: - files~=^\.mergify\.yml$ actions: delete_head_branch: + - name: notify the backport policy + conditions: + - -label~=^backport + - base=main + - -merged + - -closed + actions: + comment: + message: | + This pull request does not have a backport label. + If this is a bug or security fix, could you label this PR @{{author}}? 🙏. + For such, you'll need to label your PR with: + * The upcoming major version of the Elastic Stack + * The upcoming minor version of the Elastic Stack (if you're not pushing a breaking change) + + To fixup this pull request, you need to add the backport labels for the needed + branches, such as: + * `backport-v8./d.0` is the label to automatically backport to the `8./d` branch. `/d` is the digit - name: notify the backport has not been merged yet conditions: - -merged @@ -141,15 +159,6 @@ pull_request_rules: comment: message: | This pull request has not been merged yet. Could you please review and merge it @{{ assignee | join(', @') }}? 🙏 - - name: remove-backport label - conditions: - - label~=backport-v - - -merged - - -closed - actions: - label: - remove: - - backport-skip - name: backport patches to 7.17 branch conditions: - merged @@ -163,29 +172,29 @@ pull_request_rules: labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" - - name: backport patches to 8.2 branch + - name: backport patches to 8.3 branch conditions: - merged - - label=backport-v8.2.0 + - label=backport-v8.3.0 actions: backport: assignees: - "{{ author }}" branches: - - "8.2" + - "8.3" labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" - - name: backport patches to 8.3 branch + - name: backport patches to 8.4 branch conditions: - merged - - label=backport-v8.3.0 + - label=backport-v8.4.0 actions: backport: assignees: - "{{ author }}" branches: - - "8.3" + - "8.4" labels: - "backport" title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index e69b47b6274..940ec20ee7f 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -76,6 +76,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Drop event batch when get HTTP status 413 from Elasticsearch to avoid infinite loop {issue}14350[14350] {pull}29368[29368] - Allow to use metricbeat for named mssql instances. {issue}24076[24076] {pull}30859[30859] - Setting DEV=true when running `mage build` now correctly generates binaries without optimisations and with debug symbols {pull}31955[31955] +- The beat.cgroup.memory.mem.usage.bytes metric is now a gauge {issue}31582[31582] {pull}32652[32652] ==== Added diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 53de2f793f0..60febf52d44 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,125 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.4.0]] +=== Beats version 8.4.0 +https://github.com/elastic/beats/compare/v8.3.3\...v8.4.0[View commits] + +==== Breaking changes + +*Heartbeat* +- Browser monitors (beta) now write to the `synthetics-*` index prefix. {pull}32064[32064] +- Setting a custom index for a given monitor is now deprecated. Streams are preferred. {pull}32064[32064] +- Browser monitors now default to a max concurrency of two. {pull}32564[32564] + +==== Bugfixes + +*Affecting all Beats* + +- Fix namespacing for agent self-monitoring, CPU no longer reports as zero. {pull}32336[32336] +- Expand fields in `decode_json_fields` if target is set. {issue}31712[31712] {pull}32010[32010] + +*Auditbeat* + +- auditd module: Fix parsing of audit rules where arguments are quoted (like file paths containing spaces). {pull}32421[32421] +- auditd module: Fix minimum AuditStatus length so that library can support kernels from 2.6.32. {pull}32421[32421] +- system/socket: Reduce memory usage of the dataset. {issue}32191[32191] {pull}32192[32192] + +*Filebeat* + +- Fix counter for number of events published in `httpjson` input. {pull}31993[31993] +- Fix handling of Checkpoint event for R81. {issue}32380[32380] {pull}32458[32458] +- gcp-pubsub input: Restart Pub/Sub client on all errors. {issue}32550[32550] {pull}32712[32712] + +*Heartbeat* + +- Send targetted error message for unexpected synthetics exits. {pull}31936[31936] +- Reduced memory usage slightly for browser monitors. {pull}32317[32317] +- Automatically kill zombie-ish node processes. {pull}32393[32393] +- Added timeout for browser monitors. {pull}32434[32434] +- Fix bug with browser jobs that had missing check groups or sent empty events. {pull}32542[32542] + +*Metricbeat* + +- Update Kubernetes apiserver metricset to not collect deprecated metrics and fix dashboard. {pull}31973[31973] +- Check for nil metadata in GCP. {pull}32281[32281] +- Update Kubernetes controllermanager metricset to not collect deprecated metrics and fix dashboard. {pull}32037[32037] +- Fix ARN parsing for Cloudwatch resource names with leading slashes. {pull}32358[32358] +- Fix an infinite loop in AWS billing metricset. {pull}32626[32626] +- Add missing metrics in AWS Transit Gateway module {pull}32617[32617] +- Replace internal expiring cache used by the Kubernetes module with in-memory dictionary. {pull}32539[32539] +- Oracle Module: Refactor module to use existing host parsers instead of doing its own parsing of hosts. {issue}31611[31611] {pull}31692[#31692] +- Oracle Module: Correctly handle special characters in the connection string. {issue}24609[24609] {pull}31368[#31368] + +*Winlogbeat* + +- Powershell: Fix processing of parameter details. {pull}31833[31833] +- Security: Fix processing of sidlist, access list and access mask. {pull}31833[31833] +- Fix fatal invalid memory write on Windows 11. {issue}32469[32469] {pull}32519[32519] +- Fix handling of event formatting when no metadata is available on Windows 11. {issue}32468[32468] {pull}32519[32519] + +==== Added + +*Affecting all Beats* + +- Improve performance of disk queue by coalescing writes. {pull}31935[31935] + +*Auditbeat* + +- Add `immutable` option to the auditd module. {issue}8352[8352] {pull}32381[32381] + +*Filebeat* + +- Add `auth.oauth2.google.jwt_json` option to `httpjson` input. {pull}31750[31750] +- Add authentication fields to RabbitMQ module documents. {issue}31159[31159] {pull}31680[31680] +- Add template helper function for decoding hexadecimal strings. {pull}31886[31886] +- Add new `parser` called `include_message` to filter based on message contents. {issue}31794[31794] {pull}32094[32094] +- Extend list of mapped record types in o365 Audit module. {pull}32217[32217] +- Add references for CRI-O configuration in input-container and in our Kubernetes manifests. {issue}32149[32149] {pull}32151[32151] +- httpjson input: Add `replaceAll` helper function to template context. {pull}32365[32365] +- Optimize grok patterns in system.auth module pipeline. {pull}32360[32360] +- Checkpoint module: add authentication operation outcome enrichment. {issue}32230[32230] {pull}32431[32431] +- Add documentation for decode_xml_wineventlog processor field mappings. {pull}32456[32456] + +*Metricbeat* + +- Oracle Module: New sysmetric metricset. {issue}30946[30946] {pull}31462[#31462] +- AWS Fargate: Added support for DesiredStatus and KnownStatus. {issue}32077[32077] {pull}32342[#32342] +- Enable Generic SQL merge metrics to a single event for sql_queries using new flag. {pull}32394[32394] +- Add distribution type metrics for GCP. {pull}32170[32170] + +*Packetbeat* + +- Add support for specifying default route interface sniffing. {issue}31905[31905] {pull}31950[31950] +- Add support for TCP transport to the SIP protocol. {issue}28166[28166] {pull}32346[32346] + +[[release-notes-8.3.3]] +=== Beats version 8.3.3 +https://github.com/elastic/beats/compare/v8.3.2\...v8.3.3[View commits] + +==== Bugfixes + +*Affecting all Beats* + +- Fix OS name reported by add_host_metadata on Windows 11. {issue}30833[30833] {pull}32259[32259] +- Fix race condition when reloading running inputs. {pull}32309[32309] + +*Filebeat* + +- Fix Cisco AMP rate limit and pagination. {pull}32030[32030] +- Fix wrong state ID in states registry for awss3 s3 direct input. {pull}32164[32164] +- cisco/asa: fix handling of user names when there are Security Group Tags present. {issue}32009[32009] {pull}32196[32196] + +*Metricbeat* + +- Update elasticsearch node_stats metricset to use keyword for cgroup memory instead of long. {pull}32197[32197] + +==== Added + +*Metricbeat* + +- Azure Billing: upgrade Usage Details API to version 2019-10-01. {pull}31970[31970] + [[release-notes-8.3.2]] === Beats version 8.3.2 https://github.com/elastic/beats/compare/v8.3.1\...v8.3.2[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c5ae22d824c..333e5997a8d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -10,19 +10,19 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Affecting all Beats* -- Fix namespacing on self-monitoring {pull}32336[32336] +* Upgrade to Go 1.18. Certificates signed with SHA-1 are now rejected. See the Go 1.18 https://tip.golang.org/doc/go1.18#sha1[release notes] for details. {pull}32493[32493] + +- Fix namespacing on self-monitoring {pull}32336[32336] +- Fix formatting of hardware addresses populated by the add-host-metadata processor. {issue}32264[32264] {pull}32265[32265] *Auditbeat* *Filebeat* - + - [Httpjson]- Fix for pagination at root level not working when used with chaining. *Heartbeat* -- Browser monitors (beta) no write to the `synthetics-*` index prefix. {pull}32064[32064] -- Setting a custom index for a given monitor is now deprecated. Streams are preferred. {pull}32064[32064] - *Metricbeat* @@ -40,47 +40,58 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Affecting all Beats* +- Fix namespacing for agent self-monitoring, CPU no longer reports as zero. {pull}32336[32336] +- Fix namespacing on self-monitoring {pull}32336[32336] - Expand fields in `decode_json_fields` if target is set. {issue}31712[31712] {pull}32010[32010] -- Fix OS name reported by add_host_metadata on Windows 11. {issue}30833[30833] {pull}32259[32259] -- Fix race condition when reloading runners {pull}32309[32309] - Fix race condition when stopping runners {pull}32433[32433] +- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491] *Auditbeat* - auditd module: Fix parsing of audit rules where arguments are quoted (like file paths containing spaces). {pull}32421[32421] - auditd module: Fix minimum AuditStatus length so that library can support kernels from 2.6.32. {pull}32421[32421] - system/socket: Reduce memory usage of the dataset. {issue}32191[32191] {pull}32192[32192] +- Fix rendering of MAC addresses to conform to ECS. {issue}32621[32621] {pull}32622[32622] +- Fixes a bug with the auditd module where data is corrupted because it was not copied before the byte slice was reused. {issue}32818[32818] {pull}32823[32823] *Filebeat* -- Fix Cisco AMP rate limit and pagination. {pull}32030[32030] -- Fix wrong state ID in states registry for awss3 s3 direct input. {pull}32164[32164] -- cisco/asa: fix handling of user names when there are Security Group Tags present. {issue}32009[32009] {pull}32196[32196] -- Fix race conditions when reloading input V2 and filestream input {pull}32309[32309] - Fix counter for number of events published in `httpjson` input. {pull}31993[31993] - Fix handling of Checkpoint event for R81. {issue}32380[32380] {pull}32458[32458] +- Fix a hang on `apt-get update` stage in packaging. {pull}32580[32580] +- gcp-pubsub input: Restart Pub/Sub client on all errors. {issue}32550[32550] {pull}32712[32712] +- Fix not parsing as json when `json` and `ndjson` content types have charset information in `aws-s3` input {pull}32767[32767] +- Update `cloud.region` parsing in cloudtrail fileset. {pull}32763[32763] +- Fix file.path field in cloudtrail fileset to use json.digestS3Object. {pull}32759[32759] +- Fix rendering of MAC addresses to conform to ECS. {issue}32621[32621] {pull}32622[32622] +- Import dashboards from CEF integration. {pull}32766[32766] + +*Auditbeat* + + +*Filebeat* + +- Add handling of AAA operations for Cisco ASA module. {issue}32257[32257] {pull}32789[32789] *Heartbeat* -- Send targetted error message for unexpected synthetics exits. {pull}31936[31936] -- Reduced memory usage slightly for browser monitors. {pull}32317[32317] -- Automatically kill zombie-ish node processes. {pull}32393[32393] *Metricbeat* -- update kubernetes apiserver metricset to not collect deprecated metrics and fix dashboard {pull}31973[31973] -- Check for nil metadata in GCP. {pull}32281[32281] -- update kubernetes controllermanager metricset to not collect deprecated metrics and fix dashboard {pull}32037[32037] -- update elasticsearch node_stats metricset to use keyword for cgroup memory instead of long {pull}32197[32197] -- Fix to ARN parsing for Cloudwatch resource names with leading slashes {pull}32358[32358] +- Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] *Packetbeat* +- Fix formatting of debug logs. {pull}32698[32698] +- Fix rendering of MAC addresses to conform to ECS. {issue}32621[32621] {pull}32622[32622] *Winlogbeat* - Powershell: Fix processing of parameter details. {pull}31833[31833] - Security: Fix processing of sidlist, access list and access mask. {pull}31833[31833] +- Fix fatal invalid memory write on Windows 11. {issue}32469[32469] {pull}32519[32519] +- Fix handling of event formatting when no metadata is available on Windows 11. {issue}32468[32468] {pull}32519[32519] +- Reduce severity of message salvage failure logging. {pull}32697[32697] *Functionbeat* @@ -94,30 +105,20 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Affecting all Beats* - Improve performance of disk queue by coalescing writes. {pull}31935[31935] +- Update `elastic/go-structform` from `v0.0.9` to `v0.0.10` to reduce memory usage. {pull}32536[32536] *Auditbeat* -- Add `immutable` option to the auditd module. {issue}8352[8352] {pull}32381[32381] *Filebeat* -- Add `text/csv` decoder to `httpjson` input {pull}28564[28564] -- Update `aws-s3` input to connect to non AWS S3 buckets {issue}28222[28222] {pull}28234[28234] -- Add support for '/var/log/pods/' path for add_kubernetes_metadata processor with `resource_type: pod`. {pull}28868[28868] -- Add documentation for add_kubernetes_metadata processors `log_path` matcher. {pull}28868[28868] -- Add support for parsers on journald input {pull}29070[29070] -- Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087] -- threatintel module: Add new Recorded Future integration. {pull}30030[30030] -- Add `auth.oauth2.google.jwt_json` option to `httpjson` input. {pull}31750[31750] -- Add authentication fields to RabbitMQ module documents. {issue}31159[31159] {pull}31680[31680] -- Add template helper function for decoding hexadecimal strings. {pull}31886[31886] -- Add new `parser` called `include_message` to filter based on message contents. {issue}31794[31794] {pull}32094[32094] -- Extend list of mapped record types in o365 Audit module. {pull}32217[32217] -- Add references for CRI-O configuration in input-container and in our kubernetes manifests {issue}32149[32149] {pull}32151[32151] -- httpjson input: Add `replaceAll` helper function to template context. {pull}32365[32365] +- httpjson input: Add `toJSON` helper function to template context. {pull}32472[32472] - Optimize grok patterns in system.auth module pipeline. {pull}32360[32360] - Checkpoint module: add authentication operation outcome enrichment. {issue}32230[32230] {pull}32431[32431] - add documentation for decode_xml_wineventlog processor field mappings. {pull}32456[32456] +- httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412] +- Add cloudflare R2 to provider list in AWS S3 input. {pull}32620[32620] +- Add support for single string containing multiple relation-types in getRFC5988Link. {pull}32811[32811] *Auditbeat* @@ -130,21 +131,9 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Metricbeat* -- Enhance Oracle Module: Change tablespace metricset collection period {issue}30948[30948] {pull}31259[#31259] -- Add orchestrator cluster ECS fields in kubernetes events {pull}31341[31341] -- Enhance Oracle Module: Refactor module to properly use host parsers instead of doing its own parsing of hosts {issue}31611[31611] {pull}31692[#31692] -- Enhance Oracle Module: Connection string for Oracle does not handle special characters properly {issue}24609[24609] {pull}31368[#31368] -- Enhance Oracle Module: New sysmetric metricset {issue}30946[30946] {pull}31462[#31462] -- Upgrade Mongodb library in Beats to v5 {pull}31185[31185] -- Azure Billing: upgrade Usage Details API to version 2019-10-01 {pull}31970[31970] -* Differentiate between actual idle CPU states and an uninterruptible disk sleep. https://github.com/elastic/elastic-agent-system-metrics/pull/32[system-metrics#32] -- AWS Fargate: Added support for DesiredStatus and KnownStatus {issue}32077[32077] {pull}32342[#32342] -- Enable Generic SQL merge metrics to a single event for sql_queries using new flag {pull}32394[32394] *Packetbeat* -- Add support for specifying default route interface sniffing. {issue}31905[31905] {pull}31950[31950] -- Add support for TCP transport to the SIP protocol. {issue}28166[28166] {pull}32346[32346] *Functionbeat* @@ -190,3 +179,9 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] + + + + + + diff --git a/Jenkinsfile b/Jenkinsfile index e5ad1a7f873..1bca8803290 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -89,9 +89,6 @@ pipeline { stageStatusCache(id: 'Checks'){ withBeatsEnv(archive: false, id: "checks") { dumpVariables() - whenTrue(env.ONLY_DOCS == 'true') { - cmd(label: "make check", script: "make check") - } whenTrue(env.ONLY_DOCS == 'false') { runChecks() } @@ -224,23 +221,17 @@ def runChecks() { mapParallelTasks["${k}"] = v } } - mapParallelTasks['default'] = { - cmd(label: 'make check-default', script: 'make check-default') - } + // Run pre-commit within the current node and in Jenkins + // hence there is no need to use docker login in the GitHub actions + // some docker images are hosted in an internal docker registry. mapParallelTasks['pre-commit'] = runPreCommit() parallel(mapParallelTasks) } def runPreCommit() { return { - withNode(labels: 'ubuntu-18 && immutable', forceWorkspace: true){ - withGithubNotify(context: 'Check pre-commit', tab: 'tests') { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - preCommit(commit: "${GIT_BASE_COMMIT}", junit: true) - } - } + withGithubNotify(context: 'Check pre-commit', tab: 'tests') { + preCommit(commit: "${GIT_BASE_COMMIT}", junit: true) } } } @@ -356,6 +347,13 @@ def withTools(Map args = [:], Closure body) { withGCP() { body() } + } else if (args.get('nodejs', false)) { + withNodeJSEnv() { + withEnv(["ELASTIC_SYNTHETICS_CAPABLE=true"]) { + cmd(label: "Install @elastic/synthetics", script: "npm i -g @elastic/synthetics") + body() + } + } } else { body() } @@ -523,7 +521,7 @@ def e2e(Map args = [:]) { if (args.e2e.get('entrypoint', '')?.trim()) { e2e_with_entrypoint(args) } else { - runE2E(testMatrixFile: args.e2e?.get('testMatrixFile', ''), + runE2E(testMatrixFile: '.ci/.e2e-tests-beats.yaml', beatVersion: "${env.VERSION}-SNAPSHOT", gitHubCheckName: "e2e-${args.context}", gitHubCheckRepo: env.REPO, @@ -588,10 +586,11 @@ def targetWithoutNode(Map args = [:]) { def dockerArch = args.get('dockerArch', 'amd64') def enableRetry = args.get('enableRetry', false) def withGCP = args.get('withGCP', false) + def withNodejs = args.get('withNodejs', false) withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { dumpVariables() - withTools(k8s: installK8s, gcp: withGCP) { + withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) { // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. dir(isMage ? directory : '') { @@ -1088,6 +1087,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction { def installK8s = args.content.get('installK8s', false) def withAWS = args.content.get('withAWS', false) def withGCP = args.content.get('withGCP', false) + def withNodejs = args.content.get('withNodejs', false) // // What's the retry policy for fighting the flakiness: // 1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim @@ -1118,6 +1118,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction { withModule: withModule, isMage: true, withGCP: withGCP, + withNodejs: withNodejs, id: args.id, enableRetry: enableRetry) } diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 8142fc2e451..811b4ae985b 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -17,7 +17,6 @@ projects: - "x-pack/osquerybeat" - "x-pack/packetbeat" - "x-pack/winlogbeat" - - "dev-tools" ## Changeset macros that are defined here and used in each specific 2.0 pipeline. changeset: diff --git a/NOTICE.txt b/NOTICE.txt index 1e892284760..8f29a2b4816 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9858,11 +9858,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.2.9 +Version: v0.2.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.11/LICENSE: Apache License Version 2.0, January 2004 @@ -10069,11 +10069,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-shipper-client -Version: v0.2.0 +Version: v0.4.0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-shipper-client@v0.2.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-shipper-client@v0.4.0/LICENSE.txt: Elastic License 2.0 @@ -10172,11 +10172,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.4.2 +Version: v0.4.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.4.2/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.4.4/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -10805,11 +10805,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-libaudit/v2 -Version: v2.3.1 +Version: v2.3.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.3.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.3.2/LICENSE.txt: Apache License @@ -11440,25 +11440,214 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-lookslike@v0 -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-lumber -Version: v0.1.0 +Version: v0.1.2-0.20220819171948-335fde24ea0f Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-lumber@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-lumber@v0.1.2-0.20220819171948-335fde24ea0f/LICENSE: -Copyright (c) 2012–2016 Elasticsearch -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -------------------------------------------------------------------------------- @@ -11712,11 +11901,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-seccomp-bpf@ -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-structform -Version: v0.0.9 +Version: v0.0.10 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.10/LICENSE: Apache License Version 2.0, January 2004 @@ -20245,12 +20434,12 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/v2@v2.0.0/LICENS -------------------------------------------------------------------------------- -Dependency : go.elastic.co/go-licence-detector -Version: v0.5.0 +Dependency : go.elastic.co/ecszap +Version: v1.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@v0.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENSE: Apache License @@ -20441,7 +20630,7 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 Elastic and contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20455,44 +20644,255 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@ See the License for the specific language governing permissions and limitations under the License. - --------------------------------------------------------------------------------- -Dependency : go.etcd.io/bbolt -Version: v1.3.6 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.etcd.io/bbolt@v1.3.6/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2013 Ben Johnson - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -------------------------------------------------------------------------------- -Dependency : go.mongodb.org/mongo-driver -Version: v1.5.1 +Dependency : go.elastic.co/go-licence-detector +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.mongodb.org/mongo-driver@v1.5.1/LICENSE: +Contents of probable licence file $GOMODCACHE/go.elastic.co/go-licence-detector@v0.5.0/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : go.etcd.io/bbolt +Version: v1.3.6 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.etcd.io/bbolt@v1.3.6/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2013 Ben Johnson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : go.mongodb.org/mongo-driver +Version: v1.5.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.mongodb.org/mongo-driver@v1.5.1/LICENSE: Apache License Version 2.0, January 2004 @@ -21008,11 +21408,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.0.0-20220702020025-31831981b65f +Version: v0.0.0-20220728004956-3c1f35247d10 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20220702020025-31831981b65f/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20220728004956-3c1f35247d10/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -21901,6 +22301,36 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/gokrb5.v7@v7.5. limitations under the License. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/natefinch/lumberjack.v2 +Version: v2.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/natefinch/lumberjack.v2@v2.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Nate Finch + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v2 Version: v2.4.0 @@ -26087,6 +26517,37 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/trac limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/BurntSushi/toml +Version: v0.3.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/toml@v0.3.1/COPYING: + +The MIT License (MIT) + +Copyright (c) 2013 TOML authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/OneOfOne/xxhash Version: v1.2.2 @@ -30254,207 +30715,6 @@ Contents of probable licence file $GOMODCACHE/github.com/containerd/containerd@v limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/coreos/go-systemd -Version: v0.0.0-20190321100706-95778dfbb74e -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/coreos/go-systemd@v0.0.0-20190321100706-95778dfbb74e/LICENSE: - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/creack/pty Version: v1.1.11 @@ -38993,11 +39253,11 @@ OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/klauspost/compress -Version: v1.13.6 +Version: v1.15.9 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.13.6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.15.9/LICENSE: Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2019 Klaus Post. All rights reserved. @@ -42980,38 +43240,6 @@ Contents of probable licence file $GOMODCACHE/github.com/tklauser/numcpus@v0.3.0 --------------------------------------------------------------------------------- -Dependency : github.com/ugorji/go -Version: v1.1.8 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/ugorji/go@v1.1.8/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2012-2015 Ugorji Nwoke. -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/urso/diag Version: v0.0.0-20200210123136-21b3cc8eb797 @@ -44459,256 +44687,45 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : go.elastic.co/ecszap -Version: v1.0.1 +Dependency : go.elastic.co/fastjson +Version: v1.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE: + +Copyright 2018 Elasticsearch BV + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--- + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : go.opencensus.io +Version: v0.23.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2020 Elastic and contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- -Dependency : go.elastic.co/fastjson -Version: v1.1.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE: - -Copyright 2018 Elasticsearch BV - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ---- - -Copyright (c) 2016 Mail.Ru Group - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.opencensus.io -Version: v0.23.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.23.0/LICENSE: +Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.23.0/LICENSE: Apache License @@ -46231,207 +46248,6 @@ See the License for the specific language governing permissions and limitations under the License. --------------------------------------------------------------------------------- -Dependency : k8s.io/klog -Version: v1.0.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/k8s.io/klog@v1.0.0/LICENSE: - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : k8s.io/klog/v2 Version: v2.30.0 diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 6cd5392aa2d..b0309a6ef5b 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.12 +FROM golang:1.18.5 RUN \ apt-get update \ diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 463594ef7e0..667cd64f574 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C auditbeat check; - make -C auditbeat update; - make -C x-pack/auditbeat check; - make -C x-pack/auditbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/auditbeat/docs/modules/auditd.asciidoc b/auditbeat/docs/modules/auditd.asciidoc index 8fed93ede61..a0d2693487e 100644 --- a/auditbeat/docs/modules/auditd.asciidoc +++ b/auditbeat/docs/modules/auditd.asciidoc @@ -164,9 +164,9 @@ newer and no rules have been defined. Otherwise `unicast` will be used. This option can only be used with the `socket_type: unicast` since {beatname_uc} needs to manage the rules to be able to set it. + -It is important to note that with this setting set, {beatname_uc} should never -be stopped, as it won't be able to resume processing `auditd` events until the -system is restarted. +It is important to note that with this setting enabled, if {beatname_uc} is +stopped and resumed events will continue to be processed but the +configuration won't be updated until the system is restarted entirely. *`resolve_ids`*:: This boolean setting enables the resolution of UIDs and GIDs to their associated names. The default value is true. diff --git a/auditbeat/module/auditd/_meta/docs.asciidoc b/auditbeat/module/auditd/_meta/docs.asciidoc index c751d20042b..587a40dd982 100644 --- a/auditbeat/module/auditd/_meta/docs.asciidoc +++ b/auditbeat/module/auditd/_meta/docs.asciidoc @@ -157,9 +157,9 @@ newer and no rules have been defined. Otherwise `unicast` will be used. This option can only be used with the `socket_type: unicast` since {beatname_uc} needs to manage the rules to be able to set it. + -It is important to note that with this setting set, {beatname_uc} should never -be stopped, as it won't be able to resume processing `auditd` events until the -system is restarted. +It is important to note that with this setting enabled, if {beatname_uc} is +stopped and resumed events will continue to be processed but the +configuration won't be updated until the system is restarted entirely. *`resolve_ids`*:: This boolean setting enables the resolution of UIDs and GIDs to their associated names. The default value is true. diff --git a/auditbeat/module/auditd/audit_linux.go b/auditbeat/module/auditd/audit_linux.go index d695bee2e21..baff3363bed 100644 --- a/auditbeat/module/auditd/audit_linux.go +++ b/auditbeat/module/auditd/audit_linux.go @@ -168,7 +168,19 @@ func closeAuditClient(client *libaudit.AuditClient, log *logp.Logger) { func (ms *MetricSet) Run(reporter mb.PushReporterV2) { defer closeAuditClient(ms.client, ms.log) - if err := ms.addRules(reporter); err != nil { + // Don't attempt to change configuration if audit rules are locked (enabled == 2). + // Will result in EPERM. + status, err := ms.client.GetStatus() + if err != nil { + err = fmt.Errorf("failed to get audit status before adding rules: %w", err) + reporter.Error(err) + return + } + + if status.Enabled == auditLocked { + err := errors.New("Skipping rule configuration: Audit rules are locked") + reporter.Error(err) + } else if err := ms.addRules(reporter); err != nil { reporter.Error(err) ms.log.Errorw("Failure adding audit rules", "error", err) return @@ -181,7 +193,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) { return } - if ms.config.Immutable { + if ms.config.Immutable && status.Enabled != auditLocked { if err := ms.client.SetImmutable(libaudit.WaitForReply); err != nil { reporter.Error(err) ms.log.Errorw("Failure setting audit config as immutable", "error", err) @@ -267,18 +279,6 @@ func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error { } defer closeAuditClient(client, ms.log) - // Don't attempt to change configuration if audit rules are locked (enabled == 2). - // Will result in EPERM. - status, err := client.GetStatus() - if err != nil { - err = fmt.Errorf("failed to get audit status before adding rules: %w", err) - reporter.Error(err) - return err - } - if status.Enabled == auditLocked { - return errors.New("Skipping rule configuration: Audit rules are locked") - } - // Delete existing rules. n, err := client.DeleteRules() if err != nil { @@ -333,56 +333,60 @@ func (ms *MetricSet) initClient() error { ms.log.Infow("audit status from kernel at start", "audit_status", status) if status.Enabled == auditLocked { - return errors.New("failed to configure: The audit system is locked") - } - - if fm, _ := ms.config.failureMode(); status.Failure != fm { - if err = ms.client.SetFailure(libaudit.FailureMode(fm), libaudit.NoWait); err != nil { - return fmt.Errorf("failed to set audit failure mode in kernel: %w", err) + if !ms.config.Immutable { + return errors.New("failed to configure: The audit system is locked") } } - if status.BacklogLimit != ms.config.BacklogLimit { - if err = ms.client.SetBacklogLimit(ms.config.BacklogLimit, libaudit.NoWait); err != nil { - return fmt.Errorf("failed to set audit backlog limit in kernel: %w", err) + if status.Enabled != auditLocked { + if fm, _ := ms.config.failureMode(); status.Failure != fm { + if err = ms.client.SetFailure(libaudit.FailureMode(fm), libaudit.NoWait); err != nil { + return fmt.Errorf("failed to set audit failure mode in kernel: %w", err) + } } - } - if ms.backpressureStrategy&(bsKernel|bsAuto) != 0 { - // "kernel" backpressure mitigation strategy - // - // configure the kernel to drop audit events immediately if the - // backlog queue is full. - if status.FeatureBitmap&libaudit.AuditFeatureBitmapBacklogWaitTime != 0 { - ms.log.Info("Setting kernel backlog wait time to prevent backpressure propagating to the kernel.") - if err = ms.client.SetBacklogWaitTime(0, libaudit.NoWait); err != nil { - return fmt.Errorf("failed to set audit backlog wait time in kernel: %w", err) + if status.BacklogLimit != ms.config.BacklogLimit { + if err = ms.client.SetBacklogLimit(ms.config.BacklogLimit, libaudit.NoWait); err != nil { + return fmt.Errorf("failed to set audit backlog limit in kernel: %w", err) } - } else { - if ms.backpressureStrategy == bsAuto { - ms.log.Warn("setting backlog wait time is not supported in this kernel. Enabling workaround.") - ms.backpressureStrategy |= bsUserSpace + } + + if ms.backpressureStrategy&(bsKernel|bsAuto) != 0 { + // "kernel" backpressure mitigation strategy + // + // configure the kernel to drop audit events immediately if the + // backlog queue is full. + if status.FeatureBitmap&libaudit.AuditFeatureBitmapBacklogWaitTime != 0 { + ms.log.Info("Setting kernel backlog wait time to prevent backpressure propagating to the kernel.") + if err = ms.client.SetBacklogWaitTime(0, libaudit.NoWait); err != nil { + return fmt.Errorf("failed to set audit backlog wait time in kernel: %w", err) + } } else { - return errors.New("kernel backlog wait time not supported by kernel, but required by backpressure_strategy") + if ms.backpressureStrategy == bsAuto { + ms.log.Warn("setting backlog wait time is not supported in this kernel. Enabling workaround.") + ms.backpressureStrategy |= bsUserSpace + } else { + return errors.New("kernel backlog wait time not supported by kernel, but required by backpressure_strategy") + } } } - } - if ms.backpressureStrategy&(bsKernel|bsUserSpace) == bsUserSpace && ms.config.RateLimit == 0 { - // force a rate limit if the user-space strategy will be used without - // corresponding backlog_wait_time setting in the kernel - ms.config.RateLimit = 5000 - } + if ms.backpressureStrategy&(bsKernel|bsUserSpace) == bsUserSpace && ms.config.RateLimit == 0 { + // force a rate limit if the user-space strategy will be used without + // corresponding backlog_wait_time setting in the kernel + ms.config.RateLimit = 5000 + } - if status.RateLimit != ms.config.RateLimit { - if err = ms.client.SetRateLimit(ms.config.RateLimit, libaudit.NoWait); err != nil { - return fmt.Errorf("failed to set audit rate limit in kernel: %w", err) + if status.RateLimit != ms.config.RateLimit { + if err = ms.client.SetRateLimit(ms.config.RateLimit, libaudit.NoWait); err != nil { + return fmt.Errorf("failed to set audit rate limit in kernel: %w", err) + } } - } - if status.Enabled == 0 { - if err = ms.client.SetEnabled(true, libaudit.NoWait); err != nil { - return fmt.Errorf("failed to enable auditing in the kernel: %w", err) + if status.Enabled == 0 { + if err = ms.client.SetEnabled(true, libaudit.NoWait); err != nil { + return fmt.Errorf("failed to enable auditing in the kernel: %w", err) + } } } @@ -997,7 +1001,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) { "select the most suitable subscription method." switch c.SocketType { case unicast: - if isLocked { + if isLocked && !c.Immutable { log.Errorf("requested unicast socket_type is not available "+ "because audit configuration is locked in the kernel "+ "(enabled=2). %s", useAutodetect) @@ -1022,7 +1026,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) { // attempt to determine the optimal socket_type if hasMulticast { if hasRules { - if isLocked { + if isLocked && !c.Immutable { log.Warn("Audit rules specified in the configuration " + "cannot be applied because the audit rules have been locked " + "in the kernel (enabled=2). A multicast audit subscription " + @@ -1033,7 +1037,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) { } return multicast, nil } - if isLocked { + if isLocked && !c.Immutable { log.Errorf("Cannot continue: audit configuration is locked " + "in the kernel (enabled=2) which prevents using unicast " + "sockets. Multicast audit subscriptions are not available " + diff --git a/auditbeat/module/auditd/audit_linux_test.go b/auditbeat/module/auditd/audit_linux_test.go index dcbb0f18ffe..3beb2dfdaad 100644 --- a/auditbeat/module/auditd/audit_linux_test.go +++ b/auditbeat/module/auditd/audit_linux_test.go @@ -81,7 +81,8 @@ func TestImmutable(t *testing.T) { returnACK().returnStatus(). // Send expected ACKs for initialization // With one extra for SetImmutable - returnACK().returnACK().returnACK().returnACK().returnACK().returnACK(). + returnACK().returnStatus().returnACK().returnACK(). + returnACK().returnACK().returnACK().returnACK(). // Send one auditd message. returnMessage(userLoginFailMsg) @@ -114,7 +115,8 @@ func TestData(t *testing.T) { // Get Status response for initClient returnACK().returnStatus(). // Send expected ACKs for initialization - returnACK().returnACK().returnACK().returnACK().returnACK(). + returnACK().returnStatus().returnACK().returnACK(). + returnACK().returnACK().returnACK(). // Send three auditd messages. returnMessage(userLoginFailMsg). returnMessage(execveMsgs...). @@ -127,10 +129,10 @@ func TestData(t *testing.T) { auditMetricSet.client = &libaudit.AuditClient{Netlink: mock} events := mbtest.RunPushMetricSetV2(10*time.Second, 3, ms) + assertNoErrors(t, events) if len(events) != 3 { t.Fatalf("expected 3 events, but received %d", len(events)) } - assertNoErrors(t, events) assertFieldsAreDocumented(t, events) @@ -146,7 +148,8 @@ func TestLoginType(t *testing.T) { // Get Status response for initClient returnACK().returnStatus(). // Send expected ACKs for initialization - returnACK().returnACK().returnACK().returnACK().returnACK(). + returnACK().returnStatus().returnACK().returnACK(). + returnACK().returnACK().returnACK(). // Send an authentication failure and a success. returnMessage(userLoginFailMsg). returnMessage(userLoginSuccessMsg). diff --git a/auditbeat/module/auditd/golden_files_test.go b/auditbeat/module/auditd/golden_files_test.go index 252c2b0ec8f..2a14851cbf9 100644 --- a/auditbeat/module/auditd/golden_files_test.go +++ b/auditbeat/module/auditd/golden_files_test.go @@ -196,7 +196,8 @@ func TestGoldenFiles(t *testing.T) { // Get Status response for initClient returnACK().returnStatus(). // Send expected ACKs for initialization - returnACK().returnACK().returnACK().returnACK().returnACK(). + returnACK().returnStatus().returnACK().returnACK(). + returnACK().returnACK().returnACK(). // Send audit messages returnMessage(lines...). // Send stream terminator diff --git a/deploy/kubernetes/heartbeat-kubernetes.yaml b/deploy/kubernetes/heartbeat-kubernetes.yaml index 9352a50983b..1998ae56e37 100644 --- a/deploy/kubernetes/heartbeat-kubernetes.yaml +++ b/deploy/kubernetes/heartbeat-kubernetes.yaml @@ -100,7 +100,7 @@ spec: runAsUser: 0 resources: limits: - memory: 1536mi + memory: 1536Mi requests: # for synthetics, 2 full cores is a good starting point for relatively consistent perform of a single concurrent check # For lightweight checks as low as 100m is fine diff --git a/deploy/kubernetes/heartbeat/heartbeat-deployment.yaml b/deploy/kubernetes/heartbeat/heartbeat-deployment.yaml index 63855464aff..3f1a73d3324 100644 --- a/deploy/kubernetes/heartbeat/heartbeat-deployment.yaml +++ b/deploy/kubernetes/heartbeat/heartbeat-deployment.yaml @@ -46,7 +46,7 @@ spec: runAsUser: 0 resources: limits: - memory: 1536mi + memory: 1536Mi requests: # for synthetics, 2 full cores is a good starting point for relatively consistent perform of a single concurrent check # For lightweight checks as low as 100m is fine diff --git a/dev-tools/Jenkinsfile.yml b/dev-tools/Jenkinsfile.yml deleted file mode 100644 index e0efd075817..00000000000 --- a/dev-tools/Jenkinsfile.yml +++ /dev/null @@ -1,17 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "^dev-tools/.*" - - "^libbeat/scripts/Makefile" - comments: ## when PR comment contains any of those entries - - "/test dev-tools" - labels: ## when PR labels matches any of those entries - - "dev-tools" - parameters: ## when parameter was selected in the UI. - - "dev-tools" - tags: true ## for all the tags -platform: "immutable && ubuntu-18" ## default label for all the stages -stages: - checks: - make: "make -C dev-tools check" - stage: checks diff --git a/dev-tools/kubernetes/.gitignore b/dev-tools/kubernetes/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/dev-tools/kubernetes/.gitignore @@ -0,0 +1 @@ +/build diff --git a/dev-tools/kubernetes/README.md b/dev-tools/kubernetes/README.md new file mode 100644 index 00000000000..4860b74c269 --- /dev/null +++ b/dev-tools/kubernetes/README.md @@ -0,0 +1,84 @@ +# Readme + +This folder container some dev tools that make it easier to develop and deploy filebeat and metricbeat running inside a Kubernetes cluster. This is especially useful when developing the metricbeat module `kubernetes` since it requires metricbeat to run inside a Kubernetes cluster in order to interact with kube-state-metrics and the Kubernetes APIs. + +In details, a combination of Dockerfiles, Kubernetes manifests and Tiltfile make it possible to have features like: +- hot reloading of code running in Kubernetes, without re-applying the Kubernetes manifest +- remote debugging (with breakpoints) both metricbeat/filebeat running as a pod in a Kind Kubernetes cluster + +[Tilt](https://tilt.dev/) is a dev toolkit for microservices. + + +## Setup +You can install Tilt by using the command + +```shell +curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash +``` + + +## How to run +The Tiltfile that orchestrates everything is located at `dev-tools/kubernetes/Tiltfile`. All the following commands need to be run from `dev-tools/kubernetes`, in the same folder where the Tiltfile is located. + +How to run Tilt + +```shell +tilt up +``` + +This will open a terminal and optionally a web UI where you can interact with Tilt, see container logs and restart resources. + +Once you are done with Tilt, you can simply `CTRL+C` from the open Tilt terminal. The resources that you started in k8s will still be running though. + +For more information on how to configure Tilt to run different scenarios look at the comments in the Tiltfile. + +If you want to remove all the k8s resources that you started with Tilt, you can run + +```shell +tilt down +``` + +If you want to use a remote debugger with Visual Studio code, you need to provide a `.vscode/launch.json` similar to the following file. In order for this to work on your laptop, you need to replace `` with the absolute path of the root folder in this project. This file is currently not under git because it depends on the user configuration, it is only useful for VisualStudio Code and in a folder usually ignored by git. + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Connect to server", + "type": "go", + "request": "attach", + "mode": "remote", + "debugAdapter": "dlv-dap", + "port": 56268, + "host": "127.0.0.1", + "showLog": true, + "trace": "trace", + "cwd": "${workspaceFolder}", + "substitutePath": [ + { + "from": "${workspaceFolder}", + "to": "" + } + ] + } + + ] +} +``` + + +## Run vs debug mode +The behavior of the Tiltfile can be changed by calling the function `beat()` with different parameters: +- `beat`: `metricbeat` to test Metricbeat, `filebeat` to test Filebeat +- `mode`: `debug` to start a remote debugger that you can connect to from your IDE with hot reloading enabled, `run` to just run Metricbeat without a debugger but still with hot reloading enabled +- `arch`: `amd64` to build go binary for amd64 architecture, `arm64` to build go binary for arm64 (aka M1 Apple chip) architecture +- `k8s_env`: `kind` to run against a Kind cluster with no docker registry, `gcp` to use a docker registry on GCP. More info on docker registry on GCP at https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper. +- `k8s_cluster`: `single` to use a single node k8s cluster, `multi` to use a k8s with more than 1 node. + if running on a multi-node cluster we expect to have at least 2 workers and a control plane node. + A Beat in debugger mode will run on a node with a Tain `debugger=yes:NoSchedule`, while 1 Beat per node will run on all the other worker nodes. + More info on Taints and Tolerations at https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/. + You can add a taint with the following command: + `kubectl taint nodes debugger=yes:NoSchedule` + +You can modify the Tiltfile while `tilt up` is running in the background. Tilt will try its best to update everything in place but depending what changes you made, you might want to `tilt down` and `tilt up` again just to make sure that everything was updated correctly. diff --git a/dev-tools/kubernetes/Tiltfile b/dev-tools/kubernetes/Tiltfile new file mode 100644 index 00000000000..d38da951ff9 --- /dev/null +++ b/dev-tools/kubernetes/Tiltfile @@ -0,0 +1,199 @@ + +load('ext://restart_process', 'docker_build_with_restart') + + +def compile( + beat="metricbeat", + mode="debug", + arch="arm64", +): + flags = "" + suffix = "" + if mode == "debug": + flags = '-gcflags=all="-N -l"' + suffix = "-debugger" + build_cmd = "CGO_ENABLED=0 GOOS=linux GOARCH={} go build {} -o build/{}{} ../../{}".format( + arch, flags, beat, suffix, beat) + print("Build cmd: {}".format(build_cmd)) + + resource_name = '{}-compile{}'.format(beat, suffix) + print("Resource name: {}".format(resource_name)) + + local_resource( + resource_name, + build_cmd, + deps=['.'], + ignore=[ + "build", + "Tiltfile" + ]) + + +def build( + beat="metricbeat", + mode="run", + k8s_env="kind", +): + docker_image = '{}-{}-image'.format(beat, mode) + if k8s_env == "gcp": + # In order to push to GCR you need + # gcloud auth login + # gcloud auth configure-docker + # + # More info at https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper + docker_registry = "us.gcr.io/elastic-observability".format( + docker_image) + + default_registry(docker_registry) + print("Docker registry: {}".format(docker_registry)) + + print("Docker image: {}".format(docker_image)) + + docker_file = '{}/Dockerfile.{}'.format(beat, mode) + print("Docker file: {}".format(docker_file)) + + suffix = "" + if mode == "run": + docker_entrypoint = [ + '/usr/share/{}/{}{}'.format(beat, beat, suffix), + ] + else: + suffix = "-debugger" + docker_entrypoint = [ + "dlv", + "--headless", + "--listen=:56268", + "--api-version=2", + "--log", + "--log-output", + "debugger", + "exec", + "/usr/share/{}/{}{}".format(beat, beat, suffix), + "--" + ] + print("Docker entrypoint: {}".format(docker_entrypoint)) + + docker_build_with_restart( + docker_image, + '.', + entrypoint=docker_entrypoint, + dockerfile=docker_file, + only=["build"], + live_update=[ + sync('./build/{}{}', '/usr/share/{}'.format(beat, suffix, beat)), + ], + ) + + +def k8s_deploy( + beat="metricbeat", + mode="run", + k8s_cluster="single", +): + + if k8s_cluster == "multi": + k8s_manifest = '{}/manifest.{}.{}.yaml'.format(beat, mode, k8s_cluster) + else: + k8s_manifest = '{}/manifest.{}.yaml'.format(beat, mode) + print("K8s manifest: {}".format(k8s_manifest)) + + k8s_yaml(k8s_manifest) + + +def k8s_expose( + beat="metricbeat", + mode="run", + k8s_cluster="single", +): + if mode == "run": + k8s_resource(beat) + elif mode == "debug" and k8s_cluster == "multi": + runner_resource = "{}-runner".format(beat) + k8s_resource(runner_resource) + + debugger_resource = "{}-debugger".format(beat) + k8s_resource(debugger_resource, port_forwards=56268) + else: + k8s_resource(beat, port_forwards=56268) + + allow_k8s_contexts(k8s_context()) + + +# Beat() is used to test beats in k8s cluster + +# Parameters: +# `beat`: `metricbeat` to test Metricbeat, `filebeat` to test Filebeat +# `mode`: `debug` to start a remote debugger that you can connect to from your IDE with hot reloading enabled, `run` to just run Metricbeat without a debugger but still with hot reloading enabled +# `arch`: `amd64` to build go binary for amd64 architecture, `arm64` to build go binary for arm64 (aka M1 Apple chip) architecture +# `k8s_env`: `kind` to run against a Kind cluster with no docker repo, `gcp` to use a docker repo on GCP +# `k8s_cluster`: `single` to use a single node k8s cluster, `multi` to use a k8s with more than 1 node. +# if running on a multi-node cluster we expect to have at least 2 workers and a control plane node. One of the workers (eg. worker1) +# should have a taint and a label (for node affinity) to make sure that only the debugger runs on that node. You need to run the following commands: +# kubectl taint nodes worker1 debugger=ok:NoSchedule +# kubectl label nodes worker1 debugger=ok +# More info on Taints and Tolerations at https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/. +# More on node affinity at https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity. +def beat( + beat="metricbeat", + mode="run", + arch="arm64", + k8s_env="kind", + k8s_cluster="single", + **kwargs): + if beat not in ["metricbeat", "filebeat"]: + print("Invalid beat: {}".format(beat)) + exit(-1) + if mode not in ["run", "debug"]: + print("Invalid mode: {}".format(mode)) + exit(-1) + if arch not in ["arm64", "amd64"]: + print("Invalid arch: {}".format(arch)) + exit(-1) + if k8s_env not in ["kind", "gcp"]: + print("Invalid k8s_env: {}".format(k8s_env)) + exit(-1) + if k8s_cluster not in ["single", "multi"]: + print("Invalid k8s_cluster: {}".format(k8s_cluster)) + exit(-1) + + print("Mode: {}".format(mode)) + print("Beat: {}".format(beat)) + print("Arch: {}".format(arch)) + print("K8s_env: {}".format(k8s_env)) + print("K8s_cluster: {}".format(k8s_cluster)) + + if k8s_cluster == "multi" and mode == "run": + print("K8s_cluter=multi and mode=run is not supported yet") + exit(-1) + + if beat == "filebeat" and k8s_cluster == "multi": + print("K8s_cluter=multi and beat=filebeat is not supported yet") + exit(-1) + + # Compile + if k8s_cluster == "multi": + compile(beat=beat, mode="debug", arch=arch) + compile(beat=beat, mode="run", arch=arch) + else: + compile(beat=beat, mode=mode, arch=arch) + + # Docker Build + if k8s_cluster == "multi": + build(beat=beat, mode="debug", k8s_env=k8s_env) + build(beat=beat, mode="run", k8s_env=k8s_env) + else: + build(beat=beat, mode=mode, k8s_env=k8s_env) + + # Deploy to k8s + k8s_deploy(beat=beat, mode=mode, k8s_cluster=k8s_cluster) + + # ## Expose resource + k8s_expose(beat=beat, mode=mode, k8s_cluster=k8s_cluster) + + +beat(beat="metricbeat", + mode="debug", + arch="arm64", + k8s_env="kind", + k8s_cluster="multi", + ) diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug new file mode 100644 index 00000000000..661ac535c40 --- /dev/null +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -0,0 +1,21 @@ +FROM golang:alpine3.15 as builder + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin + +ENV CGO_ENABLED=0 + +RUN go install github.com/go-delve/delve/cmd/dlv@v1.9.0 + +COPY build/filebeat-debugger /usr/share/filebeat/filebeat-debugger + +FROM alpine:3.15 + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin + +WORKDIR /usr/share/filebeat + +COPY --from=builder /go/bin/dlv /go/bin/dlv +COPY --from=builder /usr/share/filebeat/filebeat-debugger /usr/share/filebeat/filebeat-debugger + +ENTRYPOINT ["dlv", "--headless", "--listen=:56268", "--api-version=2", "--log", "--log-output", "debugger", "exec", "/usr/share/filebeat/filebeat-debugger", "--"] +CMD [ "-e" ] diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.run b/dev-tools/kubernetes/filebeat/Dockerfile.run new file mode 100644 index 00000000000..f020df4770f --- /dev/null +++ b/dev-tools/kubernetes/filebeat/Dockerfile.run @@ -0,0 +1,11 @@ +FROM alpine:3.15 + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin + +WORKDIR /usr/share/filebeat + +COPY build/filebeat /usr/share/filebeat/filebeat + +ENTRYPOINT ["./filebeat"] + +CMD [ "-e" ] diff --git a/dev-tools/kubernetes/filebeat/manifest.debug.yaml b/dev-tools/kubernetes/filebeat/manifest.debug.yaml new file mode 100644 index 00000000000..36600e5bf5c --- /dev/null +++ b/dev-tools/kubernetes/filebeat/manifest.debug.yaml @@ -0,0 +1,242 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: + #filebeat.autodiscover: + # providers: + # - type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log + + processors: + - add_cloud_metadata: + - add_host_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + protocol: https + ssl.verification_mode: "none" + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + allow_older_versions: true +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + selector: + matchLabels: + k8s-app: filebeat + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: filebeat + image: filebeat-debug-image + args: [ + "-c", "/etc/filebeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + ports: + - containerPort: 56268 + hostPort: 56268 + protocol: TCP + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + ## on debugger image cause pod to crash since we use > 350MB of memory + # limits: + # memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0640 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart + - name: data + hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +- apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat + # should be the namespace where filebeat is running + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- diff --git a/dev-tools/kubernetes/filebeat/manifest.run.yaml b/dev-tools/kubernetes/filebeat/manifest.run.yaml new file mode 100644 index 00000000000..70e4612aee1 --- /dev/null +++ b/dev-tools/kubernetes/filebeat/manifest.run.yaml @@ -0,0 +1,237 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + + # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: + #filebeat.autodiscover: + # providers: + # - type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + # hints.default_config: + # type: container + # paths: + # - /var/log/containers/*${data.kubernetes.container.id}.log + + processors: + - add_cloud_metadata: + - add_host_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + protocol: https + ssl.verification_mode: "none" + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + allow_older_versions: true +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + selector: + matchLabels: + k8s-app: filebeat + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: filebeat + image: filebeat-run-image + args: [ + "-c", "/etc/filebeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0640 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart + - name: data + hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +- apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat + # should be the namespace where filebeat is running + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug new file mode 100644 index 00000000000..1462bc18b59 --- /dev/null +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -0,0 +1,21 @@ +FROM golang:alpine3.15 as builder + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin + +ENV CGO_ENABLED=0 + +RUN go install github.com/go-delve/delve/cmd/dlv@v1.9.0 + +COPY build/metricbeat-debugger /usr/share/metricbeat/metricbeat-debugger + +FROM alpine:3.15 + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin + +WORKDIR /usr/share/metricbeat + +COPY --from=builder /go/bin/dlv /go/bin/dlv +COPY --from=builder /usr/share/metricbeat/metricbeat-debugger /usr/share/metricbeat/metricbeat-debugger + +ENTRYPOINT ["dlv", "--headless", "--listen=:56268", "--api-version=2", "--log", "--log-output", "debugger", "exec", "/usr/share/metricbeat/metricbeat-debugger", "--"] +CMD [ "-e" ] diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.run b/dev-tools/kubernetes/metricbeat/Dockerfile.run new file mode 100644 index 00000000000..c1e0586c78a --- /dev/null +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.run @@ -0,0 +1,11 @@ +FROM alpine:3.15 + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin + +WORKDIR /usr/share/metricbeat + +COPY build/metricbeat /usr/share/metricbeat/metricbeat + +ENTRYPOINT ["./metricbeat"] + +CMD [ "-e" ] diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml new file mode 100644 index 00000000000..12f51a2a500 --- /dev/null +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml @@ -0,0 +1,467 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + # In large Kubernetes clusters consider setting unique to false + # to avoid using the leader election strategy and + # instead run a dedicated Metricbeat instance using a Deployment in addition to the DaemonSet + unique: true + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_daemonset + - state_replicaset + - state_pod + - state_container + - state_job + - state_cronjob + - state_resourcequota + - state_statefulset + - state_service + # If `https` is used to access `kube-state-metrics`, uncomment following settings: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + - module: kubernetes + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + period: 30s + # Uncomment this to get k8s events: + #- module: kubernetes + # metricsets: + # - event + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + protocol: https + ssl.verification_mode: "none" + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + allow_older_versions: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, + # remove ssl.verification_mode entry and use the CA, for instance: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + - module: kubernetes + metricsets: + - proxy + period: 10s + host: ${NODE_NAME} + hosts: ["localhost:10249"] + # If using Red Hat OpenShift should be used this `hosts` setting instead: + # hosts: ["localhost:29101"] +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat-runner + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + selector: + matchLabels: + k8s-app: metricbeat + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: metricbeat-run-image + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +# Deploy a single Metricbeat instance for debugging +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metricbeat-debugger + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + replicas: 1 + selector: + matchLabels: + app: metricbeat-debugger + template: + metadata: + labels: + app: metricbeat-debugger + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: debugger + operator: In + values: + - ok + tolerations: + - key: "debugger" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: metricbeat + image: metricbeat-debug-image + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + ports: + - containerPort: 56268 + hostPort: 56268 + protocol: TCP + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + ## on debugger image cause pod to crash since we use > 350MB of memory + # limits: + # memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] +# Enable this rule only if planing to use Kubernetes keystore +#- apiGroups: [""] +# resources: +# - secrets +# verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + - cronjobs + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml new file mode 100644 index 00000000000..7e7d6e8f2ad --- /dev/null +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml @@ -0,0 +1,362 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + # In large Kubernetes clusters consider setting unique to false + # to avoid using the leader election strategy and + # instead run a dedicated Metricbeat instance using a Deployment in addition to the DaemonSet + unique: true + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_daemonset + - state_replicaset + - state_pod + - state_container + - state_job + - state_cronjob + - state_resourcequota + - state_statefulset + - state_service + # If `https` is used to access `kube-state-metrics`, uncomment following settings: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + - module: kubernetes + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + period: 30s + # Uncomment this to get k8s events: + #- module: kubernetes + # metricsets: + # - event + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + protocol: https + ssl.verification_mode: "none" + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + allow_older_versions: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, + # remove ssl.verification_mode entry and use the CA, for instance: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + - module: kubernetes + metricsets: + - proxy + period: 10s + host: ${NODE_NAME} + hosts: ["localhost:10249"] + # If using Red Hat OpenShift should be used this `hosts` setting instead: + # hosts: ["localhost:29101"] +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + selector: + matchLabels: + k8s-app: metricbeat + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: metricbeat-debug-image + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + ports: + - containerPort: 56268 + hostPort: 56268 + protocol: TCP + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + ## on debugger image cause pod to crash since we use > 350MB of memory + # limits: + # memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] +# Enable this rule only if planing to use Kubernetes keystore +#- apiGroups: [""] +# resources: +# - secrets +# verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + - cronjobs + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- diff --git a/dev-tools/kubernetes/metricbeat/manifest.run.yaml b/dev-tools/kubernetes/metricbeat/manifest.run.yaml new file mode 100644 index 00000000000..883b4486248 --- /dev/null +++ b/dev-tools/kubernetes/metricbeat/manifest.run.yaml @@ -0,0 +1,357 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-config + namespace: kube-system + labels: + k8s-app: metricbeat +data: + metricbeat.yml: |- + metricbeat.config.modules: + # Mounted `metricbeat-daemonset-modules` configmap: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + metricbeat.autodiscover: + providers: + - type: kubernetes + scope: cluster + node: ${NODE_NAME} + # In large Kubernetes clusters consider setting unique to false + # to avoid using the leader election strategy and + # instead run a dedicated Metricbeat instance using a Deployment in addition to the DaemonSet + unique: true + templates: + - config: + - module: kubernetes + hosts: ["kube-state-metrics:8080"] + period: 10s + add_metadata: true + metricsets: + - state_node + - state_deployment + - state_daemonset + - state_replicaset + - state_pod + - state_container + - state_job + - state_cronjob + - state_resourcequota + - state_statefulset + - state_service + # If `https` is used to access `kube-state-metrics`, uncomment following settings: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + - module: kubernetes + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + period: 30s + # Uncomment this to get k8s events: + #- module: kubernetes + # metricsets: + # - event + # To enable hints based autodiscover uncomment this: + #- type: kubernetes + # node: ${NODE_NAME} + # hints.enabled: true + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + protocol: https + ssl.verification_mode: "none" + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + allow_older_versions: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-daemonset-modules + namespace: kube-system + labels: + k8s-app: metricbeat +data: + system.yml: |- + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)' + kubernetes.yml: |- + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + host: ${NODE_NAME} + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If there is a CA bundle that contains the issuer of the certificate used in the Kubelet API, + # remove ssl.verification_mode entry and use the CA, for instance: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + - module: kubernetes + metricsets: + - proxy + period: 10s + host: ${NODE_NAME} + hosts: ["localhost:10249"] + # If using Red Hat OpenShift should be used this `hosts` setting instead: + # hosts: ["localhost:29101"] +--- +# Deploy a Metricbeat instance per node for node metrics retrieval +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +spec: + selector: + matchLabels: + k8s-app: metricbeat + template: + metadata: + labels: + k8s-app: metricbeat + spec: + serviceAccountName: metricbeat + terminationGracePeriodSeconds: 30 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: metricbeat + image: metricbeat-run-image + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/metricbeat.yml + readOnly: true + subPath: metricbeat.yml + - name: data + mountPath: /usr/share/metricbeat/data + - name: modules + mountPath: /usr/share/metricbeat/modules.d + readOnly: true + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: config + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-config + - name: modules + configMap: + defaultMode: 0640 + name: metricbeat-daemonset-modules + - name: data + hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) + path: /var/lib/metricbeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] +# Enable this rule only if planing to use Kubernetes keystore +#- apiGroups: [""] +# resources: +# - secrets +# verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + - cronjobs + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 811b0c9a6b3..8fcd1e2e6d1 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -296,6 +296,15 @@ func (b GolangCrossBuilder) Build() error { verbose = "true" } var args []string + // There's a bug on certain debian versions: + // https://discuss.linuxcontainers.org/t/debian-jessie-containers-have-extremely-low-performance/1272 + // basically, apt-get has a bug where will try to iterate through every possible FD as set by the NOFILE ulimit. + // On certain docker installs, docker will set the ulimit to a value > 10^9, which means apt-get will take >1 hour. + // This runs across all possible debian platforms, since there's no real harm in it. + if strings.Contains(image, "debian") { + args = append(args, "--ulimit", "nofile=262144:262144") + } + if runtime.GOOS != "windows" { args = append(args, "--env", "EXEC_UID="+strconv.Itoa(os.Getuid()), diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 6817cae5f6c..738d041fcba 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -160,7 +160,7 @@ USER root # Install additional fonts as well RUN for iter in {1..10}; do \ apt-get update -y && \ - $NODE_PATH/node/lib/node_modules/@elastic/synthetics/node_modules/.bin/playwright install-deps && \ + $NODE_PATH/node/lib/node_modules/@elastic/synthetics/node_modules/.bin/playwright install-deps chromium && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \ fonts-noto \ fonts-noto-cjk && \ diff --git a/dev-tools/packaging/templates/ironbank/auditbeat/hardening_manifest.yaml b/dev-tools/packaging/templates/ironbank/auditbeat/hardening_manifest.yaml index e6094bc1952..da825c86b01 100644 --- a/dev-tools/packaging/templates/ironbank/auditbeat/hardening_manifest.yaml +++ b/dev-tools/packaging/templates/ironbank/auditbeat/hardening_manifest.yaml @@ -35,7 +35,7 @@ labels: ## This value can be "opensource" or "commercial" mil.dso.ironbank.image.type: "commercial" ## Product the image belongs to for grouping multiple images - mil.dso.ironbank.product.name: "beats" + mil.dso.ironbank.product.name: "auditbeat" # List of resources to make available to the offline build context resources: diff --git a/dev-tools/packaging/templates/ironbank/filebeat/hardening_manifest.yaml b/dev-tools/packaging/templates/ironbank/filebeat/hardening_manifest.yaml index 161a4458ade..3536483c194 100644 --- a/dev-tools/packaging/templates/ironbank/filebeat/hardening_manifest.yaml +++ b/dev-tools/packaging/templates/ironbank/filebeat/hardening_manifest.yaml @@ -35,7 +35,7 @@ labels: ## This value can be "opensource" or "commercial" mil.dso.ironbank.image.type: "commercial" ## Product the image belongs to for grouping multiple images - mil.dso.ironbank.product.name: "beats" + mil.dso.ironbank.product.name: "filebeat" # List of resources to make available to the offline build context resources: diff --git a/dev-tools/packaging/templates/ironbank/metricbeat/hardening_manifest.yaml b/dev-tools/packaging/templates/ironbank/metricbeat/hardening_manifest.yaml index f625c847625..845fe0ab387 100644 --- a/dev-tools/packaging/templates/ironbank/metricbeat/hardening_manifest.yaml +++ b/dev-tools/packaging/templates/ironbank/metricbeat/hardening_manifest.yaml @@ -35,7 +35,7 @@ labels: ## This value can be "opensource" or "commercial" mil.dso.ironbank.image.type: "commercial" ## Product the image belongs to for grouping multiple images - mil.dso.ironbank.product.name: "beats" + mil.dso.ironbank.product.name: "metricbeat" # List of resources to make available to the offline build context resources: diff --git a/dev-tools/packaging/templates/ironbank/packetbeat/hardening_manifest.yaml b/dev-tools/packaging/templates/ironbank/packetbeat/hardening_manifest.yaml index 6ee29922d59..3fd74f952e6 100644 --- a/dev-tools/packaging/templates/ironbank/packetbeat/hardening_manifest.yaml +++ b/dev-tools/packaging/templates/ironbank/packetbeat/hardening_manifest.yaml @@ -35,7 +35,7 @@ labels: ## This value can be "opensource" or "commercial" mil.dso.ironbank.image.type: "commercial" ## Product the image belongs to for grouping multiple images - mil.dso.ironbank.product.name: "beats" + mil.dso.ironbank.product.name: "packetbeat" # List of resources to make available to the offline build context resources: diff --git a/dev-tools/templates/.golangci.yml b/dev-tools/templates/.golangci.yml deleted file mode 100644 index e2efca97bd6..00000000000 --- a/dev-tools/templates/.golangci.yml +++ /dev/null @@ -1,145 +0,0 @@ -# options for analysis running -run: - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 15m - -issues: - # Maximum count of issues with the same text. - # Set to 0 to disable. - # Default: 3 - max-same-issues: 3 - # Maximum issues count per one linter. - # Set to 0 to disable. - # Default: 50 - max-issues-per-linter: 50 - exclude-rules: - # Exclude package name contains '-' issue because we have at least one package with - # it on its name. - - text: "ST1003:" - linters: - - stylecheck - # From mage we are priting to the console to ourselves - - path: (.*magefile.go|.*dev-tools/mage/.*) - linters: forbidigo - -output: - sort-results: true - -# Find the whole list here https://golangci-lint.run/usage/linters/ -linters: - disable-all: true - enable: - - errcheck # checking for unchecked errors in go programs - - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13. - - forbidigo # forbids identifiers matched by reg exps - - gosimple # linter for Go source code that specializes in simplifying a code - - misspell # finds commonly misspelled English words in comments - - nakedret # finds naked returns in functions greater than a specified function length - - nolintlint # reports ill-formed or insufficient nolint directives - - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks - - stylecheck # a replacement for golint - - unused # checks Go code for unused constants, variables, functions and types - - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string - - ineffassign # detects when assignments to existing variables are not used - - structcheck # finds unused struct fields - - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code - - varcheck # Finds unused global variables and constants - - asciicheck # simple linter to check that your code does not contain non-ASCII identifiers - - bodyclose # checks whether HTTP response body is closed successfully - - durationcheck # check for two durations multiplied together - - exportloopref # checks for pointers to enclosing loop variables - - goimports # Goimports does everything that gofmt does. Additionally it checks unused imports - - gosec # inspects source code for security problems - - importas # enforces consistent import aliases - - nilerr # finds the code that returns nil even if it checks that the error is not nil. - - noctx # noctx finds sending http request without context.Context - - unconvert # Remove unnecessary type conversions - - wastedassign # wastedassign finds wasted assignment statements. - - gomodguard # check for blocked dependencies - -# all available settings of specific linters -linters-settings: - errcheck: - # report about not checking of errors in type assertions: `a := b.(MyStruct)`; - check-type-assertions: false - # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`. - check-blank: false - # List of functions to exclude from checking, where each entry is a single function to exclude. - # See https://github.com/kisielk/errcheck#excluding-functions for details. - exclude-functions: - - (mapstr.M).Delete # Only returns ErrKeyNotFound, can safely be ignored. - - (mapstr.M).Put # Can only fail on type conversions, usually safe to ignore. - - errorlint: - # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats - errorf: true - # Check for plain type assertions and type switches - asserts: true - # Check for plain error comparisons - comparison: true - - forbidigo: - # Forbid the following identifiers - forbid: - - fmt.Print.* # too much log noise - # Exclude godoc examples from forbidigo checks. Default is true. - exclude_godoc_examples: true - - goimports: - local-prefixes: github.com/elastic - - gomodguard: - blocked: - # List of blocked modules. - modules: - # Blocked module. - - github.com/pkg/errors: - # Recommended modules that should be used instead. (Optional) - recommendations: - - errors - - fmt - reason: "This package is deprecated, use fmt.Errorf with %%w instead" - - gosimple: - # Select the Go version to target. The default is '1.13'. - go: "{{.GoVersion}}" - - nakedret: - # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 - max-func-lines: 0 - - nolintlint: - # Enable to ensure that nolint directives are all used. Default is true. - allow-unused: false - # Disable to ensure that nolint directives don't have a leading space. Default is true. - allow-leading-space: false - # Exclude following linters from requiring an explanation. Default is []. - allow-no-explanation: [] - # Enable to require an explanation of nonzero length after each nolint directive. Default is false. - require-explanation: true - # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. - require-specific: false - - staticcheck: - # Select the Go version to target. The default is '1.13'. - go: "{{.GoVersion}}" - checks: ["all"] - - stylecheck: - # Select the Go version to target. The default is '1.13'. - go: "{{.GoVersion}}" - # Disabled: - # ST1005: error strings should not be capitalized - checks: ["all", "-ST1005"] - - unused: - # Select the Go version to target. The default is '1.13'. - go: "{{.GoVersion}}" - - gosec: - excludes: - - G306 # Expect WriteFile permissions to be 0600 or less - - G404 # Use of weak random number generator - - G401 # Detect the usage of DES, RC4, MD5 or SHA1: Used in non-crypto contexts. - - G501 # Import blocklist: crypto/md5: Used in non-crypto contexts. - - G505 # Import blocklist: crypto/sha1: Used in non-crypto contexts. diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 05a91f1b435..4ba771195fe 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C filebeat check; - make -C filebeat update; - make -C x-pack/filebeat check; - make -C x-pack/filebeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index fcb04a67ab9..868e8ea4a4d 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -23,8 +23,6 @@ import ( "strings" "time" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/filebeat/channel" cfg "github.com/elastic/beats/v7/filebeat/config" "github.com/elastic/beats/v7/filebeat/fileset" @@ -48,8 +46,6 @@ import ( "github.com/elastic/elastic-agent-libs/monitoring" "github.com/elastic/go-concert/unison" - _ "github.com/elastic/beats/v7/filebeat/include" - // Add filebeat level processors _ "github.com/elastic/beats/v7/filebeat/processor/add_kubernetes_metadata" _ "github.com/elastic/beats/v7/libbeat/processors/decode_csv_fields" @@ -91,7 +87,7 @@ func New(plugins PluginFactory) beat.Creator { func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Beater, error) { config := cfg.DefaultConfig if err := rawConfig.Unpack(&config); err != nil { - return nil, fmt.Errorf("Error reading config file: %v", err) + return nil, fmt.Errorf("Error reading config file: %w", err) } if err := cfgwarn.CheckRemoved6xSettings( @@ -130,7 +126,7 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea if !config.ConfigInput.Enabled() && !config.ConfigModules.Enabled() && !haveEnabledInputs && config.Autodiscover == nil && !b.Manager.Enabled() { if !b.InSetupCmd { - return nil, errors.New("no modules or inputs enabled and configuration reloading disabled. What files do you want me to watch?") + return nil, fmt.Errorf("no modules or inputs enabled and configuration reloading disabled. What files do you want me to watch?") } // in the `setup` command, log this only as a warning @@ -138,7 +134,7 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea } if *once && config.ConfigInput.Enabled() && config.ConfigModules.Enabled() { - return nil, errors.New("input configs and -once cannot be used together") + return nil, fmt.Errorf("input configs and -once cannot be used together") } if config.IsInputEnabled("stdin") && len(enabledInputs) > 1 { @@ -291,7 +287,9 @@ func (fb *Filebeat) Run(b *beat.Beat) error { } var inputTaskGroup unison.TaskGroup - defer inputTaskGroup.Stop() + defer func() { + _ = inputTaskGroup.Stop() + }() if err := v2InputLoader.Init(&inputTaskGroup, v2.ModeRun); err != nil { logp.Err("Failed to initialize the input managers: %v", err) return err @@ -316,7 +314,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error { // Start the registrar err = registrar.Start() if err != nil { - return fmt.Errorf("Could not start registrar: %v", err) + return fmt.Errorf("Could not start registrar: %w", err) } // Stopping registrar will write last state @@ -340,7 +338,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error { err = crawler.Start(fb.pipeline, config.ConfigInput, config.ConfigModules) if err != nil { crawler.Stop() - return fmt.Errorf("Failed to start crawler: %+v", err) + return fmt.Errorf("Failed to start crawler: %w", err) } // If run once, add crawler completion check as alternative to done signal @@ -433,7 +431,7 @@ func newPipelineLoaderFactory(esConfig *conf.C) fileset.PipelineLoaderFactory { pipelineLoaderFactory := func() (fileset.PipelineLoader, error) { esClient, err := eslegclient.NewConnectedClient(esConfig, "Filebeat") if err != nil { - return nil, errors.Wrap(err, "Error creating Elasticsearch client") + return nil, fmt.Errorf("Error creating Elasticsearch client: %w", err) } return esClient, nil } diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 66af34c3469..51a6eb4ee11 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -56,6 +56,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -87575,6 +87576,23 @@ alias to: event.duration -- +[[exported-fields-lumberjack]] +== Lumberjack fields + +Fields from Lumberjack input. + + + +*`lumberjack`*:: ++ +-- +Structured data received in an event sent over the Lumberjack protocol. + + +type: flattened + +-- + [[exported-fields-microsoft]] == Microsoft fields diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 89695f91ae0..60c8cd9b54e 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -87,8 +87,9 @@ following command enables the +{modulename}+ module config: include::{libbeat-dir}/tab-widgets/enable-modules-widget.asciidoc[] -- -. In the module config under `modules.d`, enable the desired datasets and -change the module settings to match your environment. **Datasets are disabled by default.** +. In the module config under `modules.d`, change the module settings to match +your environment. You must enable at least one fileset in the module. +**Filesets are disabled by default.** + For example, log locations are set based on the OS. If your logs aren't in default locations, set the `paths` variable: diff --git a/filebeat/docs/include/configuring-intro.asciidoc b/filebeat/docs/include/configuring-intro.asciidoc index 9d8af1d11a1..f4f62e9a0f0 100644 --- a/filebeat/docs/include/configuring-intro.asciidoc +++ b/filebeat/docs/include/configuring-intro.asciidoc @@ -5,3 +5,6 @@ You can further refine the behavior of the +{modulename}+ module by specifying <<{modulename}-settings,variable settings>> in the +modules.d/{modulename}.yml+ file, or overriding settings at the command line. + +You must enable at least one fileset in the module. +**Filesets are disabled by default.** diff --git a/go.mod b/go.mod index ef0ef2fed13..3f9c364f951 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.17 +go 1.18 require ( cloud.google.com/go/bigquery v1.8.0 @@ -75,13 +75,13 @@ require ( github.com/eclipse/paho.mqtt.golang v1.3.5 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 github.com/elastic/go-concert v0.2.0 - github.com/elastic/go-libaudit/v2 v2.3.1 + github.com/elastic/go-libaudit/v2 v2.3.2 github.com/elastic/go-licenser v0.4.0 github.com/elastic/go-lookslike v0.3.0 - github.com/elastic/go-lumber v0.1.0 + github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 github.com/elastic/go-seccomp-bpf v1.2.0 - github.com/elastic/go-structform v0.0.9 + github.com/elastic/go-structform v0.0.10 github.com/elastic/go-sysinfo v1.8.1 github.com/elastic/go-ucfg v0.8.6 github.com/elastic/go-windows v1.0.1 // indirect @@ -159,7 +159,7 @@ require ( github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v1.0.3 - go.elastic.co/ecszap v1.0.1 // indirect + go.elastic.co/ecszap v1.0.1 go.elastic.co/go-licence-detector v0.5.0 go.etcd.io/bbolt v1.3.6 go.uber.org/atomic v1.9.0 @@ -171,7 +171,7 @@ require ( golang.org/x/net v0.0.0-20220225172249-27dd8689420f golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 - golang.org/x/sys v0.0.0-20220702020025-31831981b65f + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac golang.org/x/tools v0.1.9 @@ -202,9 +202,9 @@ require ( github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.2.1 - github.com/elastic/elastic-agent-libs v0.2.9 - github.com/elastic/elastic-agent-shipper-client v0.2.0 - github.com/elastic/elastic-agent-system-metrics v0.4.2 // do not upgrade until https://github.com/elastic/beats/issues/32467 is fixed + github.com/elastic/elastic-agent-libs v0.2.11 + github.com/elastic/elastic-agent-shipper-client v0.4.0 + github.com/elastic/elastic-agent-system-metrics v0.4.4 github.com/elastic/go-elasticsearch/v8 v8.2.0 github.com/pierrec/lz4/v4 v4.1.15 github.com/shirou/gopsutil/v3 v3.21.12 @@ -212,6 +212,7 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.0.0 go.elastic.co/apm/v2 v2.0.0 go.mongodb.org/mongo-driver v1.5.1 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) require ( @@ -293,7 +294,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.15.8 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/compress v1.15.9 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/markbates/pkger v0.17.0 // indirect github.com/mattn/go-isatty v0.0.14 // indirect diff --git a/go.sum b/go.sum index e6107647f2d..8a1b6e8f89c 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,7 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -489,7 +490,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -597,15 +597,13 @@ github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUd github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= -github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= -github.com/elastic/elastic-agent-libs v0.2.7/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= -github.com/elastic/elastic-agent-libs v0.2.9 h1:7jOCqNqEWG0kJb3fa8/SC6beSiys1TmAylH9+hWTnrM= -github.com/elastic/elastic-agent-libs v0.2.9/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= -github.com/elastic/elastic-agent-shipper-client v0.2.0 h1:p+5ep48YCOe+3nICeWmiLwQV11yDLad2n4NunI66Shg= -github.com/elastic/elastic-agent-shipper-client v0.2.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q= -github.com/elastic/elastic-agent-system-metrics v0.4.2 h1:tM24imnCLNrgrO74myzSF6RhJ3ikeF8VIxKdXB5RHzk= -github.com/elastic/elastic-agent-system-metrics v0.4.2/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI= +github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= +github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= +github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk= +github.com/elastic/elastic-agent-shipper-client v0.4.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q= +github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0= +github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI= github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE= github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= @@ -615,27 +613,27 @@ github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa2 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38= github.com/elastic/go-elasticsearch/v8 v8.2.0 h1:oagGcb1gqxT7yWpQ3E7wMP3NhGRamsKVd7kRdbuI+/Y= github.com/elastic/go-elasticsearch/v8 v8.2.0/go.mod h1:yY52i2Vj0unLz+N3Nwx1gM5LXwoj3h2dgptNGBYkMLA= -github.com/elastic/go-libaudit/v2 v2.3.1 h1:BCPhB3BZTl2YBPJy3XHIu4DFfbbbWEmLMLlK/aJ7U/0= -github.com/elastic/go-libaudit/v2 v2.3.1/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo= +github.com/elastic/go-libaudit/v2 v2.3.2 h1:qWNcA3nkwNEGh1UBDbDTVF55KR6SM1W2Ji1LGDqFEpw= +github.com/elastic/go-libaudit/v2 v2.3.2/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo= github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI= github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-lookslike v0.3.0 h1:HDI/DQ65V85ZqM7D/sbxcK2wFFnh3+7iFvBk2v2FTHs= github.com/elastic/go-lookslike v0.3.0/go.mod h1:AhH+rdJux5RlVjs+6ej4jkvYyoNRkj2crxmqeHlj3hA= -github.com/elastic/go-lumber v0.1.0 h1:HUjpyg36v2HoKtXlEC53EJ3zDFiDRn65d7B8dBHNius= -github.com/elastic/go-lumber v0.1.0/go.mod h1:8YvjMIRYypWuPvpxx7WoijBYdbB7XIh/9FqSYQZTtxQ= +github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f h1:TsPpU5EAwlt7YZoupKlxZ093qTZYdGou3EhfTF1U0B4= +github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f/go.mod h1:HHaWnZamYKWsR9/eZNHqRHob8iQDKnchHmmskT/SKko= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 h1:q8n4QjcLa4q39Q3fqHRknTBXBtegjriHFrB42YKgXGI= github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595/go.mod h1:s09U1b4P1ZxnKx2OsqY7KlHdCesqZWIhyq0Gs/QC/Us= github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f h1:FvsqAVIFZtJtK+koSvFU+/KoNQo1m14kgV5qJ8ImN+U= github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f/go.mod h1:OPGqFNdTS34kMReS5hPFtBhD9J8itmSDurs1ix2wx7c= github.com/elastic/go-seccomp-bpf v1.2.0 h1:K5fToUAMzm0pmdlYORmw0FP0DloRa1SfqRYkum647Yk= github.com/elastic/go-seccomp-bpf v1.2.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8= -github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o= github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= +github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= +github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.8.1 h1:4Yhj+HdV6WjbCRgGdZpPJ8lZQlXZLKDAeIkmQ/VRvi4= github.com/elastic/go-sysinfo v1.8.1/go.mod h1:JfllUnzoQV/JRYymbH3dO1yggI3mV2oTKSXsDHM+uIM= github.com/elastic/go-ucfg v0.7.0/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/go-ucfg v0.8.4/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= @@ -1221,8 +1219,8 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -1700,7 +1698,6 @@ github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.8 h1:/D9x7IRpfMHDlizVOgxrag5Fh+/NY+LtI8bsr+AswRA= github.com/ugorji/go v1.1.8/go.mod h1:0lNM99SwWUIRhCXnigEMClngXBk/EmpTXa7mgiewYWA= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.8 h1:4dryPvxMP9OtkjIbuNeK2nb27M38XMHLGlfNSNph/5s= @@ -2166,11 +2163,9 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220702020025-31831981b65f h1:xdsejrW/0Wf2diT5CPp3XmKUNbr7Xvw8kYilQ+6qjRY= -golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2502,6 +2497,7 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -2582,7 +2578,6 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index a857f7883b1..78730227991 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.12 +FROM golang:1.18.5 RUN \ apt-get update \ diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 98e024d73b7..87fce6bf9e0 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C heartbeat check; - make -C heartbeat update; - make -C x-pack/heartbeat check; - make -C x-pack/heartbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go index 3610534a3b2..5e0f1ff9b6e 100644 --- a/heartbeat/beater/heartbeat.go +++ b/heartbeat/beater/heartbeat.go @@ -23,6 +23,11 @@ import ( "syscall" "time" + "github.com/elastic/beats/v7/libbeat/publisher/pipeline" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/beats/v7/heartbeat/config" "github.com/elastic/beats/v7/heartbeat/hbregistry" "github.com/elastic/beats/v7/heartbeat/monitors" @@ -33,8 +38,6 @@ import ( "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/management" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" _ "github.com/elastic/beats/v7/heartbeat/security" ) @@ -67,23 +70,36 @@ func New(b *beat.Beat, rawConfig *conf.C) (beat.Beater, error) { } jobConfig := parsedConfig.Jobs - scheduler := scheduler.Create(limit, hbregistry.SchedulerRegistry, location, jobConfig, parsedConfig.RunOnce) + sched := scheduler.Create(limit, hbregistry.SchedulerRegistry, location, jobConfig, parsedConfig.RunOnce) + + pipelineClientFactory := func(p beat.Pipeline) (pipeline.ISyncClient, error) { + if parsedConfig.RunOnce { + client, err := pipeline.NewSyncClient(logp.L(), p, beat.ClientConfig{}) + if err != nil { + return nil, fmt.Errorf("could not create pipeline sync client for run_once: %w", err) + } + return client, nil + } else { + client, err := p.Connect() + return monitors.SyncPipelineClientAdaptor{C: client}, err + } + } bt := &Heartbeat{ done: make(chan struct{}), config: parsedConfig, - scheduler: scheduler, + scheduler: sched, // dynamicFactory is the factory used for dynamic configs, e.g. autodiscover / reload - dynamicFactory: monitors.NewFactory(b.Info, scheduler.Add, plugin.GlobalPluginsReg, parsedConfig.RunOnce), + dynamicFactory: monitors.NewFactory(b.Info, sched.Add, plugin.GlobalPluginsReg, pipelineClientFactory), } return bt, nil } // Run executes the beat. func (bt *Heartbeat) Run(b *beat.Beat) error { - logp.Info("heartbeat is running! Hit CTRL-C to stop it.") + logp.L().Info("heartbeat is running! Hit CTRL-C to stop it.") groups, _ := syscall.Getgroups() - logp.Info("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups) + logp.L().Info("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups) // It is important this appear before we check for run once mode // In run once mode we depend on these monitors being loaded, but not other more @@ -96,7 +112,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { if bt.config.RunOnce { bt.scheduler.WaitForRunOnce() - logp.Info("Ending run_once run") + logp.L().Info("Ending run_once run") return nil } @@ -108,7 +124,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { bt.monitorReloader = cfgfile.NewReloader(b.Publisher, bt.config.ConfigMonitors) defer bt.monitorReloader.Stop() - err := bt.RunReloadableMonitors(b) + err := bt.RunReloadableMonitors() if err != nil { return err } @@ -134,7 +150,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { <-bt.done - logp.Info("Shutting down.") + logp.L().Info("Shutting down.") return nil } @@ -145,7 +161,7 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) (stop func(), err error) { created, err := bt.dynamicFactory.Create(b.Publisher, cfg) if err != nil { if errors.Is(err, monitors.ErrMonitorDisabled) { - logp.Info("skipping disabled monitor: %s", err) + logp.L().Info("skipping disabled monitor: %s", err) continue // don't stop loading monitors just because they're disabled } @@ -166,14 +182,14 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) (stop func(), err error) { // RunCentralMgmtMonitors loads any central management configured configs. func (bt *Heartbeat) RunCentralMgmtMonitors(b *beat.Beat) { - monitors := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher) - reload.Register.MustRegisterList(b.Info.Beat+".monitors", monitors) + mons := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher) + reload.Register.MustRegisterList(b.Info.Beat+".monitors", mons) inputs := cfgfile.NewRunnerList(management.DebugK, bt.dynamicFactory, b.Publisher) reload.Register.MustRegisterList("inputs", inputs) } // RunReloadableMonitors runs the `heartbeat.config.monitors` portion of the yaml config if present. -func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) { +func (bt *Heartbeat) RunReloadableMonitors() (err error) { // Check monitor configs if err := bt.monitorReloader.Check(bt.dynamicFactory); err != nil { logp.Error(fmt.Errorf("error loading reloadable monitors: %w", err)) @@ -187,7 +203,7 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) { // makeAutodiscover creates an autodiscover object ready to be started. func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) { - autodiscover, err := autodiscover.NewAutodiscover( + ad, err := autodiscover.NewAutodiscover( "heartbeat", b.Publisher, bt.dynamicFactory, @@ -198,7 +214,7 @@ func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, if err != nil { return nil, err } - return autodiscover, nil + return ad, nil } // Stop stops the beat. diff --git a/heartbeat/config/config.go b/heartbeat/config/config.go index 93ee0b7556b..324f069d83a 100644 --- a/heartbeat/config/config.go +++ b/heartbeat/config/config.go @@ -32,7 +32,7 @@ type Config struct { ConfigMonitors *conf.C `config:"config.monitors"` Scheduler Scheduler `config:"scheduler"` Autodiscover *autodiscover.Config `config:"autodiscover"` - Jobs map[string]JobLimit `config:"jobs"` + Jobs map[string]*JobLimit `config:"jobs"` } type JobLimit struct { @@ -46,4 +46,10 @@ type Scheduler struct { } // DefaultConfig is the canonical instantiation of Config. -var DefaultConfig = Config{} +var DefaultConfig = Config{ + Jobs: map[string]*JobLimit{ + "browser": { + Limit: 2, + }, + }, +} diff --git a/heartbeat/docs/running-on-docker.asciidoc b/heartbeat/docs/running-on-docker.asciidoc index dbfcce5b489..2347e937d73 100644 --- a/heartbeat/docs/running-on-docker.asciidoc +++ b/heartbeat/docs/running-on-docker.asciidoc @@ -1 +1,13 @@ include::{libbeat-dir}/shared-docker.asciidoc[] + +[float] +==== Required network capabilities + +Under Docker, {beatname_uc} runs as a non-root user, but requires some privileged +network capabilities to operate correctly. Ensure that the +NET_RAW+ +capability is available to the container. + +["source","sh",subs="attributes"] +---- +docker run --cap-add=NET_RAW {dockerimage} +---- \ No newline at end of file diff --git a/heartbeat/docs/running-on-kubernetes.asciidoc b/heartbeat/docs/running-on-kubernetes.asciidoc index 87b5c4bb395..528f0c0b45b 100644 --- a/heartbeat/docs/running-on-kubernetes.asciidoc +++ b/heartbeat/docs/running-on-kubernetes.asciidoc @@ -74,3 +74,22 @@ $ kubectl --namespace=kube-system get deployment/{beatname_lc} NAME READY UP-TO-DATE AVAILABLE AGE {beatname_lc} 1/1 1 1 1m ------------------------------------------------ + +[float] +==== Running {beatname_uc} as unprivileged user + +Under Kubernetes, {beatname_uc} can run as a non-root user, but requires some privileged +network capabilities to operate correctly. Ensure that the +NET_RAW+ +capability is available to the container. + +["source","yaml",subs="attributes"] +---- +containers: +- name: heartbeat + image: {dockerimage} + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + add: [ NET_RAW ] +---- diff --git a/heartbeat/ecserr/ecserr.go b/heartbeat/ecserr/ecserr.go index fc1a1422217..6755abeae40 100644 --- a/heartbeat/ecserr/ecserr.go +++ b/heartbeat/ecserr/ecserr.go @@ -19,6 +19,7 @@ package ecserr import ( "fmt" + "time" ) // ECSErr represents an error per the ECS specification @@ -72,3 +73,19 @@ func NewBadCmdStatusErr(exitCode int, cmd string) *ECSErr { fmt.Sprintf("command '%s' exited unexpectedly with code: %d", cmd, exitCode), ) } + +func NewCmdTimeoutStatusErr(timeout time.Duration, cmd string) *ECSErr { + return NewECSErr( + ETYPE_IO, + "CMD_TIMEOUT", + fmt.Sprintf("command '%s' did not exit before extended timeout: %s", cmd, timeout.String()), + ) +} + +func NewSyntheticsCmdCouldNotStartErr(reason error) *ECSErr { + return NewECSErr( + ETYPE_IO, + "SYNTHETICS_CMD_COULD_NOT_START", + fmt.Sprintf("could not start command not found: %s", reason), + ) +} diff --git a/heartbeat/hbtestllext/isdefs.go b/heartbeat/hbtestllext/isdefs.go index 1ec55d19949..9e029cca885 100644 --- a/heartbeat/hbtestllext/isdefs.go +++ b/heartbeat/hbtestllext/isdefs.go @@ -41,3 +41,11 @@ var IsInt64 = isdef.Is("positiveInt64", func(path llpath.Path, v interface{}) *l } return llresult.ValidResult(path) }) + +var IsUint16 = isdef.Is("positiveUInt16", func(path llpath.Path, v interface{}) *llresult.Results { + _, ok := v.(uint16) + if !ok { + return llresult.SimpleResult(path, false, "expected a uint16") + } + return llresult.ValidResult(path) +}) diff --git a/heartbeat/monitors/factory.go b/heartbeat/monitors/factory.go index e5de12ba27e..25b62684358 100644 --- a/heartbeat/monitors/factory.go +++ b/heartbeat/monitors/factory.go @@ -21,6 +21,10 @@ import ( "fmt" "sync" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/heartbeat/monitors/plugin" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/scheduler" @@ -31,24 +35,24 @@ import ( "github.com/elastic/beats/v7/libbeat/processors/actions" "github.com/elastic/beats/v7/libbeat/processors/add_data_stream" "github.com/elastic/beats/v7/libbeat/processors/add_formatted_index" + "github.com/elastic/beats/v7/libbeat/publisher/pipeline" "github.com/elastic/beats/v7/libbeat/publisher/pipetool" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" ) // RunnerFactory that can be used to create cfg.Runner cast versions of Monitor // suitable for config reloading. type RunnerFactory struct { - info beat.Info - addTask scheduler.AddTask - byId map[string]*Monitor - mtx *sync.Mutex - pluginsReg *plugin.PluginsReg - logger *logp.Logger - runOnce bool + info beat.Info + addTask scheduler.AddTask + byId map[string]*Monitor + mtx *sync.Mutex + pluginsReg *plugin.PluginsReg + logger *logp.Logger + pipelineClientFactory PipelineClientFactory } +type PipelineClientFactory func(pipeline beat.Pipeline) (pipeline.ISyncClient, error) + type publishSettings struct { // Fields and tags to add to monitor. EventMetadata mapstr.EventMetadata `config:",inline"` @@ -69,20 +73,36 @@ type publishSettings struct { } // NewFactory takes a scheduler and creates a RunnerFactory that can create cfgfile.Runner(Monitor) objects. -func NewFactory(info beat.Info, addTask scheduler.AddTask, pluginsReg *plugin.PluginsReg, runOnce bool) *RunnerFactory { +func NewFactory(info beat.Info, addTask scheduler.AddTask, pluginsReg *plugin.PluginsReg, pcf PipelineClientFactory) *RunnerFactory { return &RunnerFactory{ - info: info, - addTask: addTask, - byId: map[string]*Monitor{}, - mtx: &sync.Mutex{}, - pluginsReg: pluginsReg, - logger: logp.L(), - runOnce: runOnce, + info: info, + addTask: addTask, + byId: map[string]*Monitor{}, + mtx: &sync.Mutex{}, + pluginsReg: pluginsReg, + logger: logp.L(), + pipelineClientFactory: pcf, } } +type NoopRunner struct{} + +func (NoopRunner) String() string { + return "" +} + +func (NoopRunner) Start() { +} + +func (NoopRunner) Stop() { +} + // Create makes a new Runner for a new monitor with the given Config. func (f *RunnerFactory) Create(p beat.Pipeline, c *conf.C) (cfgfile.Runner, error) { + if !c.Enabled() { + return NoopRunner{}, nil + } + c, err := stdfields.UnnestStream(c) if err != nil { return nil, err @@ -119,9 +139,13 @@ func (f *RunnerFactory) Create(p beat.Pipeline, c *conf.C) (cfgfile.Runner, erro } }() } - monitor, err := newMonitor(c, f.pluginsReg, p, f.addTask, safeStop, f.runOnce) + pc, err := f.pipelineClientFactory(p) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create pipeline client via factory: %w", err) + } + monitor, err := newMonitor(c, f.pluginsReg, pc, f.addTask, safeStop) + if err != nil { + return nil, fmt.Errorf("factory could not create monitor: %w", err) } if mon, ok := f.byId[monitor.stdFields.ID]; ok { @@ -137,6 +161,9 @@ func (f *RunnerFactory) Create(p beat.Pipeline, c *conf.C) (cfgfile.Runner, erro // CheckConfig checks to see if the given monitor config is valid. func (f *RunnerFactory) CheckConfig(config *conf.C) error { + if !config.Enabled() { + return nil + } return checkMonitorConfig(config, plugin.GlobalPluginsReg) } diff --git a/heartbeat/monitors/factory_test.go b/heartbeat/monitors/factory_test.go index 817c780d847..8ba16f52377 100644 --- a/heartbeat/monitors/factory_test.go +++ b/heartbeat/monitors/factory_test.go @@ -20,17 +20,16 @@ package monitors import ( "regexp" "testing" - "time" + + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/heartbeat/scheduler" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/processors/add_data_stream" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/monitoring" ) var binfo = beat.Info{ @@ -160,18 +159,39 @@ func TestPreProcessors(t *testing.T) { } } +func TestDisabledMonitor(t *testing.T) { + confMap := map[string]interface{}{ + "type": "test", + "enabled": "false", + } + + conf, err := config.NewConfigFrom(confMap) + require.NoError(t, err) + + reg, built, closed := mockPluginsReg() + f, sched, fClose := makeMockFactory(reg) + defer fClose() + defer sched.Stop() + runner, err := f.Create(&MockPipeline{}, conf) + require.NoError(t, err) + require.IsType(t, runner, NoopRunner{}) + + require.Equal(t, 0, built.Load()) + require.Equal(t, 0, closed.Load()) +} + func TestDuplicateMonitorIDs(t *testing.T) { serverMonConf := mockPluginConf(t, "custom", "custom", "@every 1ms", "http://example.net") badConf := mockBadPluginConf(t, "custom") reg, built, closed := mockPluginsReg() - pipelineConnector := &MockPipelineConnector{} + mockPipeline := &MockPipeline{} - sched := scheduler.Create(1, monitoring.NewRegistry(), time.Local, nil, false) + f, sched, fClose := makeMockFactory(reg) + defer fClose() defer sched.Stop() - f := NewFactory(binfo, sched.Add, reg, false) makeTestMon := func() (*Monitor, error) { - mIface, err := f.Create(pipelineConnector, serverMonConf) + mIface, err := f.Create(mockPipeline, serverMonConf) if mIface == nil { return nil, err } else { @@ -180,7 +200,7 @@ func TestDuplicateMonitorIDs(t *testing.T) { } // Ensure that an error is returned on a bad config - _, m0Err := newMonitor(badConf, reg, pipelineConnector, sched.Add, nil, false) + _, m0Err := newMonitor(badConf, reg, mockPipeline.ConnectSync(), sched.Add, nil) require.Error(t, m0Err) // Would fail if the previous newMonitor didn't free the monitor.id diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks.go similarity index 64% rename from heartbeat/monitors/mocks_test.go rename to heartbeat/monitors/mocks.go index 5c36284576e..5063824f134 100644 --- a/heartbeat/monitors/mocks_test.go +++ b/heartbeat/monitors/mocks.go @@ -22,7 +22,9 @@ import ( "regexp" "sync" "testing" + "time" + "github.com/gofrs/uuid" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/heartbeat/eventext" @@ -30,9 +32,12 @@ import ( "github.com/elastic/beats/v7/heartbeat/hbtestllext" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/plugin" + "github.com/elastic/beats/v7/heartbeat/scheduler" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/atomic" - conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/beats/v7/libbeat/publisher/pipeline" + beatversion "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/monitoring" "github.com/elastic/go-lookslike" @@ -40,24 +45,75 @@ import ( "github.com/elastic/go-lookslike/validator" ) -type MockBeatClient struct { - publishes []beat.Event - closed bool - mtx sync.Mutex +func makeMockFactory(pluginsReg *plugin.PluginsReg) (factory *RunnerFactory, sched *scheduler.Scheduler, close func()) { + id, _ := uuid.NewV4() + eid, _ := uuid.NewV4() + info := beat.Info{ + Beat: "heartbeat", + IndexPrefix: "heartbeat", + Version: beatversion.GetDefaultVersion(), + ElasticLicensed: true, + Name: "heartbeat", + Hostname: "localhost", + ID: id, + EphemeralID: eid, + FirstStart: time.Now(), + StartTime: time.Now(), + Monitoring: struct { + DefaultUsername string + }{ + DefaultUsername: "test", + }, + } + + sched = scheduler.Create( + 1, + monitoring.NewRegistry(), + time.Local, + nil, + true, + ) + + return NewFactory(info, sched.Add, pluginsReg, func(pipeline beat.Pipeline) (pipeline.ISyncClient, error) { + c, _ := pipeline.Connect() + return SyncPipelineClientAdaptor{C: c}, nil + }), + sched, + sched.Stop } -func (c *MockBeatClient) Publish(e beat.Event) { +type mockClient struct { + publishLog []*beat.Event + pipeline beat.Pipeline + closed bool + mtx sync.Mutex +} + +func (c *mockClient) IsClosed() bool { + c.mtx.Lock() + defer c.mtx.Unlock() + + return c.closed +} + +func (c *mockClient) Publish(e beat.Event) { c.PublishAll([]beat.Event{e}) } -func (c *MockBeatClient) PublishAll(events []beat.Event) { +func (c *mockClient) PublishAll(events []beat.Event) { c.mtx.Lock() defer c.mtx.Unlock() - c.publishes = append(c.publishes, events...) + for _, e := range events { + eLocal := e + c.publishLog = append(c.publishLog, &eLocal) + } +} + +func (c *mockClient) Wait() { } -func (c *MockBeatClient) Close() error { +func (c *mockClient) Close() error { c.mtx.Lock() defer c.mtx.Unlock() @@ -69,35 +125,51 @@ func (c *MockBeatClient) Close() error { return nil } -func (c *MockBeatClient) Publishes() []beat.Event { +func (c *mockClient) PublishedEvents() []*beat.Event { c.mtx.Lock() defer c.mtx.Unlock() - dst := make([]beat.Event, len(c.publishes)) - copy(dst, c.publishes) - return dst + return c.publishLog } -type MockPipelineConnector struct { - clients []*MockBeatClient +type MockPipeline struct { + Clients []*mockClient mtx sync.Mutex } -func (pc *MockPipelineConnector) Connect() (beat.Client, error) { +func (pc *MockPipeline) Connect() (beat.Client, error) { return pc.ConnectWith(beat.ClientConfig{}) } -func (pc *MockPipelineConnector) ConnectWith(beat.ClientConfig) (beat.Client, error) { +func (pc *MockPipeline) ConnectWith(beat.ClientConfig) (beat.Client, error) { pc.mtx.Lock() defer pc.mtx.Unlock() - c := &MockBeatClient{} + c := &mockClient{pipeline: pc} - pc.clients = append(pc.clients, c) + pc.Clients = append(pc.Clients, c) return c, nil } +// Convenience function for tests +func (pc *MockPipeline) ConnectSync() pipeline.ISyncClient { + c, _ := pc.Connect() + return SyncPipelineClientAdaptor{C: c} +} + +func (pc *MockPipeline) PublishedEvents() []*beat.Event { + pc.mtx.Lock() + defer pc.mtx.Unlock() + + var events []*beat.Event + for _, c := range pc.Clients { + events = append(events, c.PublishedEvents()...) + } + + return events +} + func baseMockEventMonitorValidator(id string, name string, status string) validator.Validator { var idMatcher isdef.IsDef if id == "" { @@ -148,7 +220,7 @@ func mockPluginBuilder() (plugin.PluginFactory, *atomic.Int, *atomic.Int) { return plugin.PluginFactory{ Name: "test", Aliases: []string{"testAlias"}, - Make: func(s string, config *conf.C) (plugin.Plugin, error) { + Make: func(s string, config *config.C) (plugin.Plugin, error) { built.Inc() // Declare a real config block with a required attr so we can see what happens when it doesn't work unpacked := struct { @@ -181,7 +253,7 @@ func mockPluginsReg() (p *plugin.PluginsReg, built *atomic.Int, closed *atomic.I return reg, built, closed } -func mockPluginConf(t *testing.T, id string, name string, schedule string, url string) *conf.C { +func mockPluginConf(t *testing.T, id string, name string, schedule string, url string) *config.C { confMap := map[string]interface{}{ "type": "test", "urls": []string{url}, @@ -194,7 +266,7 @@ func mockPluginConf(t *testing.T, id string, name string, schedule string, url s confMap["id"] = id } - conf, err := conf.NewConfigFrom(confMap) + conf, err := config.NewConfigFrom(confMap) require.NoError(t, err) return conf @@ -202,7 +274,7 @@ func mockPluginConf(t *testing.T, id string, name string, schedule string, url s // mockBadPluginConf returns a conf with an invalid plugin config. // This should fail after the generic plugin checks fail since the HTTP plugin requires 'urls' to be set. -func mockBadPluginConf(t *testing.T, id string) *conf.C { +func mockBadPluginConf(t *testing.T, id string) *config.C { confMap := map[string]interface{}{ "type": "test", "notanoption": []string{"foo"}, @@ -212,24 +284,24 @@ func mockBadPluginConf(t *testing.T, id string) *conf.C { confMap["id"] = id } - conf, err := conf.NewConfigFrom(confMap) + conf, err := config.NewConfigFrom(confMap) require.NoError(t, err) return conf } -func mockInvalidPluginConf(t *testing.T) *conf.C { +func mockInvalidPluginConf(t *testing.T) *config.C { confMap := map[string]interface{}{ "hoeutnheou": "oueanthoue", } - conf, err := conf.NewConfigFrom(confMap) + conf, err := config.NewConfigFrom(confMap) require.NoError(t, err) return conf } -func mockInvalidPluginConfWithStdFields(t *testing.T, id string, name string, schedule string) *conf.C { +func mockInvalidPluginConfWithStdFields(t *testing.T, id string, name string, schedule string) *config.C { confMap := map[string]interface{}{ "type": "test", "id": id, @@ -237,7 +309,7 @@ func mockInvalidPluginConfWithStdFields(t *testing.T, id string, name string, sc "schedule": schedule, } - conf, err := conf.NewConfigFrom(confMap) + conf, err := config.NewConfigFrom(confMap) require.NoError(t, err) return conf diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go index 61558d270e6..8d500629ece 100644 --- a/heartbeat/monitors/monitor.go +++ b/heartbeat/monitors/monitor.go @@ -21,17 +21,19 @@ import ( "fmt" "sync" + "github.com/elastic/beats/v7/libbeat/publisher/pipeline" + "github.com/mitchellh/hashstructure" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/plugin" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" "github.com/elastic/beats/v7/heartbeat/scheduler" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/publisher/pipeline" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" ) // ErrMonitorDisabled is returned when the monitor plugin is marked as disabled. @@ -43,12 +45,6 @@ const ( MON_STOPPED ) -type WrappedClient struct { - Publish func(event beat.Event) - Close func() error - wait func() -} - // Monitor represents a configured recurring monitoring configuredJob loaded from a config file. Starting it // will cause it to run with the given scheduler until Stop() is called. type Monitor struct { @@ -66,13 +62,13 @@ type Monitor struct { internalsMtx sync.Mutex close func() error - pipelineConnector beat.PipelineConnector + // pubClient accepts an ISyncClient as the lowest common denominator of client + // since async clients are a subset of sync clients + pubClient pipeline.ISyncClient // stats is the countersRecorder used to record lifecycle events // for global metrics + telemetry stats plugin.RegistryRecorder - - runOnce bool } // String prints a description of the monitor in a threadsafe way. It is important that this use threadsafe @@ -82,7 +78,7 @@ func (m *Monitor) String() string { } func checkMonitorConfig(config *conf.C, registrar *plugin.PluginsReg) error { - _, err := newMonitor(config, registrar, nil, nil, nil, false) + _, err := newMonitor(config, registrar, nil, nil, nil) return err } @@ -92,12 +88,11 @@ func checkMonitorConfig(config *conf.C, registrar *plugin.PluginsReg) error { func newMonitor( config *conf.C, registrar *plugin.PluginsReg, - pipelineConnector beat.PipelineConnector, + pubClient pipeline.ISyncClient, taskAdder scheduler.AddTask, onStop func(*Monitor), - runOnce bool, ) (*Monitor, error) { - m, err := newMonitorUnsafe(config, registrar, pipelineConnector, taskAdder, onStop, runOnce) + m, err := newMonitorUnsafe(config, registrar, pubClient, taskAdder, onStop) if m != nil && err != nil { m.Stop() } @@ -109,10 +104,9 @@ func newMonitor( func newMonitorUnsafe( config *conf.C, registrar *plugin.PluginsReg, - pipelineConnector beat.PipelineConnector, + pubClient pipeline.ISyncClient, addTask scheduler.AddTask, onStop func(*Monitor), - runOnce bool, ) (*Monitor, error) { // Extract just the Id, Type, and Enabled fields from the config // We'll parse things more precisely later once we know what exact type of @@ -122,26 +116,21 @@ func newMonitorUnsafe( return nil, err } - if !config.Enabled() { - return nil, fmt.Errorf("monitor '%s' with id '%s' skipped: %w", standardFields.Name, standardFields.ID, ErrMonitorDisabled) - } - pluginFactory, found := registrar.Get(standardFields.Type) if !found { return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", standardFields.Type, registrar.MonitorNames()) } m := &Monitor{ - stdFields: standardFields, - pluginName: pluginFactory.Name, - addTask: addTask, - configuredJobs: []*configuredJob{}, - pipelineConnector: pipelineConnector, - internalsMtx: sync.Mutex{}, - config: config, - stats: pluginFactory.Stats, - state: MON_INIT, - runOnce: runOnce, + stdFields: standardFields, + pluginName: pluginFactory.Name, + addTask: addTask, + configuredJobs: []*configuredJob{}, + pubClient: pubClient, + internalsMtx: sync.Mutex{}, + config: config, + stats: pluginFactory.Stats, + state: MON_INIT, } if m.stdFields.ID == "" { @@ -227,31 +216,7 @@ func (m *Monitor) Start() { defer m.internalsMtx.Unlock() for _, t := range m.configuredJobs { - if m.runOnce { - client, err := pipeline.NewSyncClient(logp.L(), t.monitor.pipelineConnector, beat.ClientConfig{}) - if err != nil { - logp.L().Errorf("could not start monitor: %v", err) - continue - } - t.Start(&WrappedClient{ - Publish: func(event beat.Event) { - _ = client.Publish(event) - }, - Close: client.Close, - wait: client.Wait, - }) - } else { - client, err := m.pipelineConnector.Connect() - if err != nil { - logp.L().Errorf("could not start monitor: %v", err) - continue - } - t.Start(&WrappedClient{ - Publish: client.Publish, - Close: client.Close, - wait: func() {}, - }) - } + t.Start(m.pubClient) } m.stats.StartMonitor(int64(m.endpoints)) diff --git a/heartbeat/monitors/monitor_test.go b/heartbeat/monitors/monitor_test.go index 8258f9473f9..9585920cc53 100644 --- a/heartbeat/monitors/monitor_test.go +++ b/heartbeat/monitors/monitor_test.go @@ -63,30 +63,30 @@ func TestMonitorCfgError(t *testing.T) { func testMonitorConfig(t *testing.T, conf *conf.C, eventValidator validator.Validator) { reg, built, closed := mockPluginsReg() - pipelineConnector := &MockPipelineConnector{} + pipel := &MockPipeline{} sched := scheduler.Create(1, monitoring.NewRegistry(), time.Local, nil, false) defer sched.Stop() - mon, err := newMonitor(conf, reg, pipelineConnector, sched.Add, nil, false) + mon, err := newMonitor(conf, reg, pipel.ConnectSync(), sched.Add, nil) require.NoError(t, err) mon.Start() - require.Equal(t, 1, len(pipelineConnector.clients)) - pcClient := pipelineConnector.clients[0] + require.Equal(t, 1, len(pipel.Clients)) + pcClient := pipel.Clients[0] timeout := time.Second start := time.Now() success := false for time.Since(start) < timeout && !success { - count := len(pcClient.Publishes()) + count := len(pcClient.PublishedEvents()) if count >= 1 { success = true pcClient.Close() - for _, event := range pcClient.Publishes() { + for _, event := range pcClient.PublishedEvents() { testslike.Test(t, eventValidator, event.Fields) } } else { @@ -110,12 +110,12 @@ func testMonitorConfig(t *testing.T, conf *conf.C, eventValidator validator.Vali func TestCheckInvalidConfig(t *testing.T) { serverMonConf := mockInvalidPluginConf(t) reg, built, closed := mockPluginsReg() - pipelineConnector := &MockPipelineConnector{} + pipel := &MockPipeline{} sched := scheduler.Create(1, monitoring.NewRegistry(), time.Local, nil, false) defer sched.Stop() - m, err := newMonitor(serverMonConf, reg, pipelineConnector, sched.Add, nil, false) + m, err := newMonitor(serverMonConf, reg, pipel.ConnectSync(), sched.Add, nil) require.Error(t, err) // This could change if we decide the contract for newMonitor should always return a monitor require.Nil(t, m, "For this test to work we need a nil value for the monitor.") diff --git a/metricbeat/module/kubernetes/util/metrics_cache_test.go b/heartbeat/monitors/syncpipeclient.go similarity index 54% rename from metricbeat/module/kubernetes/util/metrics_cache_test.go rename to heartbeat/monitors/syncpipeclient.go index 07b447c5fe7..461d53293b2 100644 --- a/metricbeat/module/kubernetes/util/metrics_cache_test.go +++ b/heartbeat/monitors/syncpipeclient.go @@ -15,38 +15,28 @@ // specific language governing permissions and limitations // under the License. -package util +package monitors -import ( - "testing" - "time" +import "github.com/elastic/beats/v7/libbeat/beat" - "github.com/stretchr/testify/assert" -) - -func TestValueMap(t *testing.T) { - test := newValueMap(120 * time.Second) - - // no value - assert.Equal(t, 0.0, test.Get("foo")) - - // Set and test - test.Set("foo", 3.14) - assert.Equal(t, 3.14, test.Get("foo")) +type SyncPipelineClientAdaptor struct { + C beat.Client } -func TestGetWithDefault(t *testing.T) { - test := newValueMap(120 * time.Second) +func (s SyncPipelineClientAdaptor) Publish(event beat.Event) error { + s.C.Publish(event) + return nil +} - // Empty + default - assert.Equal(t, 0.0, test.Get("foo")) - assert.Equal(t, 3.14, test.GetWithDefault("foo", 3.14)) +func (s SyncPipelineClientAdaptor) PublishAll(events []beat.Event) error { + s.C.PublishAll(events) + return nil +} - // Defined value - test.Set("foo", 38.2) - assert.Equal(t, 38.2, test.GetWithDefault("foo", 3.14)) +func (s SyncPipelineClientAdaptor) Close() error { + return s.C.Close() } -func TestContainerUID(t *testing.T) { - assert.Equal(t, "a/b/c", ContainerUID("a", "b", "c")) +func (s SyncPipelineClientAdaptor) Wait() { + // intentionally blank, async pipelines should be empty } diff --git a/heartbeat/monitors/task.go b/heartbeat/monitors/task.go index 8a5da2cf427..35563d7cd36 100644 --- a/heartbeat/monitors/task.go +++ b/heartbeat/monitors/task.go @@ -21,6 +21,8 @@ import ( "context" "fmt" + "github.com/elastic/beats/v7/libbeat/publisher/pipeline" + "github.com/elastic/beats/v7/heartbeat/eventext" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/scheduler" @@ -33,11 +35,11 @@ import ( // configuredJob represents a job combined with its config and any // subsequent processors. type configuredJob struct { - job jobs.Job - config jobConfig - monitor *Monitor - cancelFn context.CancelFunc - client *WrappedClient + job jobs.Job + config jobConfig + monitor *Monitor + cancelFn context.CancelFunc + pubClient pipeline.ISyncClient } func newConfiguredJob(job jobs.Job, config jobConfig, monitor *Monitor) *configuredJob { @@ -65,7 +67,7 @@ func (e ProcessorsError) Error() string { func (t *configuredJob) prepareSchedulerJob(job jobs.Job) scheduler.TaskFunc { return func(_ context.Context) []scheduler.TaskFunc { - return runPublishJob(job, t.client) + return runPublishJob(job, t.pubClient) } } @@ -74,20 +76,20 @@ func (t *configuredJob) makeSchedulerTaskFunc() scheduler.TaskFunc { } // Start schedules this configuredJob for execution. -func (t *configuredJob) Start(client *WrappedClient) { +func (t *configuredJob) Start(pubClient pipeline.ISyncClient) { var err error - t.client = client + t.pubClient = pubClient if err != nil { - logp.Err("could not start monitor: %v", err) + logp.L().Info("could not start monitor: %v", err) return } tf := t.makeSchedulerTaskFunc() - t.cancelFn, err = t.monitor.addTask(t.config.Schedule, t.monitor.stdFields.ID, tf, t.config.Type, client.wait) + t.cancelFn, err = t.monitor.addTask(t.config.Schedule, t.monitor.stdFields.ID, tf, t.config.Type, pubClient.Wait) if err != nil { - logp.Err("could not start monitor: %v", err) + logp.L().Info("could not start monitor: %v", err) } } @@ -96,19 +98,19 @@ func (t *configuredJob) Stop() { if t.cancelFn != nil { t.cancelFn() } - if t.client != nil { - _ = t.client.Close() + if t.pubClient != nil { + _ = t.pubClient.Close() } } -func runPublishJob(job jobs.Job, client *WrappedClient) []scheduler.TaskFunc { +func runPublishJob(job jobs.Job, pubClient pipeline.ISyncClient) []scheduler.TaskFunc { event := &beat.Event{ Fields: mapstr.M{}, } conts, err := job(event) if err != nil { - logp.Err("Job failed with: %s", err) + logp.L().Info("Job failed with: %s", err) } hasContinuations := len(conts) > 0 @@ -123,10 +125,10 @@ func runPublishJob(job jobs.Job, client *WrappedClient) []scheduler.TaskFunc { Meta: event.Meta.Clone(), Fields: event.Fields.Clone(), } - client.Publish(clone) + _ = pubClient.Publish(clone) } else { // no clone needed if no continuations - client.Publish(*event) + _ = pubClient.Publish(*event) } } @@ -142,7 +144,7 @@ func runPublishJob(job jobs.Job, client *WrappedClient) []scheduler.TaskFunc { localCont := cont contTasks[i] = func(_ context.Context) []scheduler.TaskFunc { - return runPublishJob(localCont, client) + return runPublishJob(localCont, pubClient) } } return contTasks diff --git a/heartbeat/monitors/task_test.go b/heartbeat/monitors/task_test.go index afd1ea3c565..f2fddccc4e6 100644 --- a/heartbeat/monitors/task_test.go +++ b/heartbeat/monitors/task_test.go @@ -95,12 +95,9 @@ func Test_runPublishJob(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - client := &MockBeatClient{} - queue := runPublishJob(tc.job, &WrappedClient{ - Publish: client.Publish, - Close: client.Close, - wait: func() {}, - }) + pipel := &MockPipeline{} + client := pipel.ConnectSync() + queue := runPublishJob(tc.job, client) for { if len(queue) == 0 { break @@ -110,10 +107,12 @@ func Test_runPublishJob(t *testing.T) { conts := tf(context.Background()) queue = append(queue, conts...) } - client.Close() + client.Wait() + err := client.Close() + require.NoError(t, err) - require.Len(t, client.publishes, len(tc.validators)) - for idx, event := range client.publishes { + require.Len(t, pipel.PublishedEvents(), len(tc.validators)) + for idx, event := range pipel.PublishedEvents() { testslike.Test(t, tc.validators[idx], event.Fields) } }) diff --git a/heartbeat/scheduler/schedjob_test.go b/heartbeat/scheduler/schedjob_test.go index 3269e73a486..24e6178e162 100644 --- a/heartbeat/scheduler/schedjob_test.go +++ b/heartbeat/scheduler/schedjob_test.go @@ -99,7 +99,7 @@ func TestSchedJobRun(t *testing.T) { // testRecursiveForkingJob tests that a schedJob that splits into multiple parallel pieces executes without error func TestRecursiveForkingJob(t *testing.T) { - s := Create(1000, monitoring.NewRegistry(), tarawaTime(), map[string]config.JobLimit{ + s := Create(1000, monitoring.NewRegistry(), tarawaTime(), map[string]*config.JobLimit{ "atype": {Limit: 1}, }, false) ran := batomic.NewInt(0) diff --git a/heartbeat/scheduler/scheduler.go b/heartbeat/scheduler/scheduler.go index 2439be7282d..87f25de1af5 100644 --- a/heartbeat/scheduler/scheduler.go +++ b/heartbeat/scheduler/scheduler.go @@ -72,10 +72,11 @@ type Schedule interface { RunOnInit() bool } -func getJobLimitSem(jobLimitByType map[string]config.JobLimit) map[string]*semaphore.Weighted { +func getJobLimitSem(jobLimitByType map[string]*config.JobLimit) map[string]*semaphore.Weighted { jobLimitSem := map[string]*semaphore.Weighted{} for jobType, jobLimit := range jobLimitByType { if jobLimit.Limit > 0 { + logp.L().Infof("limiting to %d concurrent jobs for '%s' type", jobLimit.Limit, jobType) jobLimitSem[jobType] = semaphore.NewWeighted(jobLimit.Limit) } } @@ -83,7 +84,7 @@ func getJobLimitSem(jobLimitByType map[string]config.JobLimit) map[string]*semap } // NewWithLocation creates a new Scheduler using the given runAt zone. -func Create(limit int64, registry *monitoring.Registry, location *time.Location, jobLimitByType map[string]config.JobLimit, runOnce bool) *Scheduler { +func Create(limit int64, registry *monitoring.Registry, location *time.Location, jobLimitByType map[string]*config.JobLimit, runOnce bool) *Scheduler { ctx, cancelCtx := context.WithCancel(context.Background()) if limit < 1 { diff --git a/heartbeat/scheduler/scheduler_test.go b/heartbeat/scheduler/scheduler_test.go index 7ba2b5baa24..a50e5cdcce8 100644 --- a/heartbeat/scheduler/scheduler_test.go +++ b/heartbeat/scheduler/scheduler_test.go @@ -245,10 +245,10 @@ func TestSchedTaskLimits(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var jobConfigByType = map[string]config.JobLimit{} + var jobConfigByType = map[string]*config.JobLimit{} jobType := "http" if tt.limit > 0 { - jobConfigByType = map[string]config.JobLimit{ + jobConfigByType = map[string]*config.JobLimit{ jobType: {Limit: tt.limit}, } } diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 6219b7bdd09..2bf7e681ae4 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,14 +12,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C libbeat check; - make -C libbeat update; - make -C x-pack/libbeat check; - make -C x-pack/libbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/libbeat/common/reload/reload.go b/libbeat/common/reload/reload.go index 099ddcb51b8..e1838619245 100644 --- a/libbeat/common/reload/reload.go +++ b/libbeat/common/reload/reload.go @@ -18,10 +18,9 @@ package reload import ( + "fmt" "sync" - "github.com/pkg/errors" - "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -72,11 +71,11 @@ func (r *Registry) Register(name string, obj Reloadable) error { defer r.Unlock() if obj == nil { - return errors.New("got a nil object") + return fmt.Errorf("got a nil object") } if r.nameTaken(name) { - return errors.Errorf("%s configuration list is already registered", name) + return fmt.Errorf("%s configuration list is already registered", name) } r.confs[name] = obj @@ -89,11 +88,11 @@ func (r *Registry) RegisterList(name string, list ReloadableList) error { defer r.Unlock() if list == nil { - return errors.New("got a nil object") + return fmt.Errorf("got a nil object") } if r.nameTaken(name) { - return errors.Errorf("%s configuration is already registered", name) + return fmt.Errorf("%s configuration is already registered", name) } r.confsLists[name] = list diff --git a/libbeat/common/reload/reload_test.go b/libbeat/common/reload/reload_test.go index 04c478f7476..c9efef19326 100644 --- a/libbeat/common/reload/reload_test.go +++ b/libbeat/common/reload/reload_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type reloadable struct{} @@ -33,7 +34,8 @@ func TestRegisterReloadable(t *testing.T) { obj := reloadable{} r := NewRegistry() - r.Register("my.reloadable", obj) + err := r.Register("my.reloadable", obj) + require.NoError(t, err) assert.Equal(t, obj, r.GetReloadable("my.reloadable")) } @@ -42,7 +44,8 @@ func TestRegisterReloadableList(t *testing.T) { objl := reloadableList{} r := NewRegistry() - r.RegisterList("my.reloadable", objl) + err := r.RegisterList("my.reloadable", objl) + require.NoError(t, err) assert.Equal(t, objl, r.GetReloadableList("my.reloadable")) } diff --git a/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc b/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc index 01653b96255..858e7c55fee 100644 --- a/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc +++ b/libbeat/docs/release-notes/breaking/breaking-8.0.asciidoc @@ -83,6 +83,16 @@ version of the operating system. * MacOS 10.13 * MacOS 10.14 +[discrete] +==== {filebeat} filesets are disabled by default +Prior to version 8.0.0, modules had some filesets that were enabled by default. +This meant that in some cases you could enable a module from the command line +and run it without modifying the configuration in the `modules.d` directory. +However this caused problems for some users. + +Starting in version 8.0.0, filesets are disabled by default. You must explicitly +enable the filesets you want {filebeat} to use. + // end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking-8.4.asciidoc b/libbeat/docs/release-notes/breaking/breaking-8.4.asciidoc new file mode 100644 index 00000000000..7a39b2e2cd5 --- /dev/null +++ b/libbeat/docs/release-notes/breaking/breaking-8.4.asciidoc @@ -0,0 +1,19 @@ +[[breaking-changes-8.4]] + +=== Breaking changes in 8.4 +++++ +8.4 +++++ + +See the <> for a complete list of breaking changes, +bug fixes, and enhancements, including changes to beta or experimental +functionality. + +//NOTE: The notable-breaking-changes tagged regions are re-used in the +//Installation and Upgrade Guide + +//tag::notable-breaking-changes[] + +There are no notable breaking changes in this release. + +// end::notable-breaking-changes[] diff --git a/libbeat/docs/release-notes/breaking/breaking.asciidoc b/libbeat/docs/release-notes/breaking/breaking.asciidoc index 67c80f54afa..665184a70c0 100644 --- a/libbeat/docs/release-notes/breaking/breaking.asciidoc +++ b/libbeat/docs/release-notes/breaking/breaking.asciidoc @@ -11,11 +11,14 @@ changes, but there are breaking changes between major versions (e.g. 7.x to See the following topics for a description of breaking changes: +* <> * <> * <> * <> * <> +include::breaking-8.4.asciidoc[] + include::breaking-8.3.asciidoc[] include::breaking-8.2.asciidoc[] diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 30943fcb8d1..0b8ae996282 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,8 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> +* <> * <> * <> * <> diff --git a/libbeat/docs/shared-docker.asciidoc b/libbeat/docs/shared-docker.asciidoc index deff7df6a27..a4a01be88a2 100644 --- a/libbeat/docs/shared-docker.asciidoc +++ b/libbeat/docs/shared-docker.asciidoc @@ -74,6 +74,7 @@ ifeval::["{beatname_lc}"=="heartbeat"] ["source", "sh", subs="attributes"] -------------------------------------------- docker run \ +--cap-add=NET_RAW \ {dockerimage} \ setup -E setup.kibana.host=kibana:5601 \ -E output.elasticsearch.hosts=["elasticsearch:9200"] <1> <2> @@ -206,6 +207,7 @@ docker run -d \ --name={beatname_lc} \ --user={beatname_lc} \ --volume="$(pwd)/{beatname_lc}.docker.yml:/usr/share/{beatname_lc}/{beatname_lc}.yml:ro" \ + --cap-add=NET_RAW \ {dockerimage} \ --strict.perms=false -e \ -E output.elasticsearch.hosts=["elasticsearch:9200"] <1> <2> diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 7d545a49658..e11194aa70c 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.4.0 :doc-branch: main -:go-version: 1.17.12 +:go-version: 1.18.5 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/monitoring/report/log/log.go b/libbeat/monitoring/report/log/log.go index 9117b3d5454..61f70d775e1 100644 --- a/libbeat/monitoring/report/log/log.go +++ b/libbeat/monitoring/report/log/log.go @@ -37,32 +37,33 @@ import ( // TODO: Replace this with a proper solution that uses the metric type from // where it is defined. See: https://github.com/elastic/beats/issues/5433 var gauges = map[string]bool{ - "libbeat.output.events.active": true, - "libbeat.pipeline.events.active": true, - "libbeat.pipeline.clients": true, - "libbeat.config.module.running": true, - "registrar.states.current": true, - "filebeat.harvester.running": true, - "filebeat.harvester.open_files": true, - "beat.memstats.memory_total": true, - "beat.memstats.memory_alloc": true, - "beat.memstats.rss": true, - "beat.memstats.gc_next": true, - "beat.info.uptime.ms": true, - "beat.cpu.user.ticks": true, - "beat.cpu.system.ticks": true, - "beat.cpu.total.value": true, - "beat.cpu.total.ticks": true, - "beat.handles.open": true, - "beat.handles.limit.hard": true, - "beat.handles.limit.soft": true, - "beat.runtime.goroutines": true, - "system.load.1": true, - "system.load.5": true, - "system.load.15": true, - "system.load.norm.1": true, - "system.load.norm.5": true, - "system.load.norm.15": true, + "libbeat.output.events.active": true, + "libbeat.pipeline.events.active": true, + "libbeat.pipeline.clients": true, + "libbeat.config.module.running": true, + "registrar.states.current": true, + "filebeat.harvester.running": true, + "filebeat.harvester.open_files": true, + "beat.memstats.memory_total": true, + "beat.memstats.memory_alloc": true, + "beat.memstats.rss": true, + "beat.memstats.gc_next": true, + "beat.info.uptime.ms": true, + "beat.cgroup.memory.mem.usage.bytes": true, + "beat.cpu.user.ticks": true, + "beat.cpu.system.ticks": true, + "beat.cpu.total.value": true, + "beat.cpu.total.ticks": true, + "beat.handles.open": true, + "beat.handles.limit.hard": true, + "beat.handles.limit.soft": true, + "beat.runtime.goroutines": true, + "system.load.1": true, + "system.load.5": true, + "system.load.15": true, + "system.load.norm.1": true, + "system.load.norm.5": true, + "system.load.norm.15": true, } // isGauge returns true when the given metric key name represents a gauge value. diff --git a/libbeat/outputs/kafka/kafka_integration_test.go b/libbeat/outputs/kafka/kafka_integration_test.go index 6f3e99ad810..ce14d22070d 100644 --- a/libbeat/outputs/kafka/kafka_integration_test.go +++ b/libbeat/outputs/kafka/kafka_integration_test.go @@ -69,7 +69,7 @@ func TestKafkaPublish(t *testing.T) { events []eventInfo }{ { - "publish single event to test topic", + "publish single event to test topic with nil config", nil, testTopic, single(mapstr.M{ @@ -186,7 +186,7 @@ func TestKafkaPublish(t *testing.T) { }), }, { - "publish single event to test topic", + "publish single event to test topic with empty config", map[string]interface{}{}, testTopic, single(mapstr.M{ @@ -199,7 +199,7 @@ func TestKafkaPublish(t *testing.T) { // that added a full 30sec to the test. Instead most tests run // in plaintext, and individual tests can switch to SCRAM // by inserting the config in this example: - "publish single event to test topic over SASL/SCRAM", + "SASL/SCRAM publish single event to test topic", map[string]interface{}{ "hosts": []string{getTestSASLKafkaHost()}, "protocol": "https", diff --git a/libbeat/outputs/shipper/README.md b/libbeat/outputs/shipper/README.md index bc1c9dd23f4..dec05449e73 100644 --- a/libbeat/outputs/shipper/README.md +++ b/libbeat/outputs/shipper/README.md @@ -19,6 +19,7 @@ output.shipper: timeout: 30 max_retries: 3 bulk_max_size: 50 + ack_polling_interval: '5ms' backoff: init: 1 max: 60 @@ -61,6 +62,12 @@ Setting `bulk_max_size` to values less than or equal to 0 disables the splitting of batches. When splitting is disabled, the queue decides on the number of events to be contained in a batch. +### `ack_polling_interval` + +The minimal interval for getting persisted index updates from the shipper server. Batches of events are acknowledged asynchronously in the background. If after the `ack_polling_interval` duration the persisted index value changed all batches pending acknowledgment will be checked against the new value and acknowledged if `persisted_index` >= `accepted_index`. + +The default value is `5ms`, cannot be set to a value less then the default. + ### `backoff.init` The number of seconds to wait before trying to republish to the shipper diff --git a/libbeat/outputs/shipper/api/shipper_mock.go b/libbeat/outputs/shipper/api/shipper_mock.go index fce9db750cd..6b26f100e67 100644 --- a/libbeat/outputs/shipper/api/shipper_mock.go +++ b/libbeat/outputs/shipper/api/shipper_mock.go @@ -19,21 +19,30 @@ package api import ( context "context" + "errors" + "time" pb "github.com/elastic/elastic-agent-shipper-client/pkg/proto" "github.com/elastic/elastic-agent-shipper-client/pkg/proto/messages" + + "github.com/gofrs/uuid" ) func NewProducerMock(cap int) *ProducerMock { + id, _ := uuid.NewV4() return &ProducerMock{ - Q: make([]*messages.Event, 0, cap), + uuid: id.String(), + Q: make([]*messages.Event, 0, cap), } } type ProducerMock struct { pb.UnimplementedProducerServer - Q []*messages.Event - Error error + Q []*messages.Event + uuid string + AcceptedCount uint32 + persistedIndex uint64 + Error error } func (p *ProducerMock) PublishEvents(ctx context.Context, r *messages.PublishRequest) (*messages.PublishReply, error) { @@ -41,6 +50,10 @@ func (p *ProducerMock) PublishEvents(ctx context.Context, r *messages.PublishReq return nil, p.Error } + if r.Uuid != p.uuid { + return nil, errors.New("UUID does not match") + } + resp := &messages.PublishReply{} for _, e := range r.Events { @@ -50,7 +63,44 @@ func (p *ProducerMock) PublishEvents(ctx context.Context, r *messages.PublishReq p.Q = append(p.Q, e) resp.AcceptedCount++ + if resp.AcceptedCount == p.AcceptedCount { + break + } } + resp.AcceptedIndex = uint64(len(p.Q)) + return resp, nil } + +func (p *ProducerMock) Persist(count uint64) { + p.persistedIndex = count +} + +func (p *ProducerMock) PersistedIndex(req *messages.PersistedIndexRequest, producer pb.Producer_PersistedIndexServer) error { + err := producer.Send(&messages.PersistedIndexReply{ + Uuid: p.uuid, + PersistedIndex: p.persistedIndex, + }) + if err != nil { + return err + } + + if !req.PollingInterval.IsValid() || req.PollingInterval.AsDuration() == 0 { + return nil + } + + ticker := time.NewTicker(req.PollingInterval.AsDuration()) + defer ticker.Stop() + + for range ticker.C { + err = producer.Send(&messages.PersistedIndexReply{ + Uuid: p.uuid, + PersistedIndex: p.persistedIndex, + }) + if err != nil { + return err + } + } + return nil +} diff --git a/libbeat/outputs/shipper/config.go b/libbeat/outputs/shipper/config.go index 7e0a5d10386..a4c71e05da2 100644 --- a/libbeat/outputs/shipper/config.go +++ b/libbeat/outputs/shipper/config.go @@ -34,21 +34,28 @@ type Config struct { // TLS/SSL configurationf or secure connection TLS *tlscommon.Config `config:"ssl"` // Timeout of a single batch publishing request - Timeout time.Duration `config:"timeout" validate:"min=1"` + Timeout time.Duration `config:"timeout" validate:"min=1"` // MaxRetries is how many times the same batch is attempted to be sent - MaxRetries int `config:"max_retries" validate:"min=-1,nonzero"` + MaxRetries int `config:"max_retries" validate:"min=-1,nonzero"` // BulkMaxSize max amount of events in a single batch BulkMaxSize int `config:"bulk_max_size"` + // AckPollingInterval is a minimal interval for getting persisted index updates from the shipper server. + // Batches of events are acknowledged asynchronously in the background. + // If after the `AckPollingInterval` duration the persisted index value changed + // all batches pending acknowledgment will be checked against the new value + // and acknowledged if `persisted_index` >= `accepted_index`. + AckPollingInterval time.Duration `config:"ack_polling_interval" validate:"min=5ms"` // Backoff strategy for the shipper output Backoff backoffConfig `config:"backoff"` } func defaultConfig() Config { return Config{ - TLS: nil, - Timeout: 30 * time.Second, - MaxRetries: 3, - BulkMaxSize: 50, + TLS: nil, + Timeout: 30 * time.Second, + MaxRetries: 3, + BulkMaxSize: 50, + AckPollingInterval: 5 * time.Millisecond, Backoff: backoffConfig{ Init: 1 * time.Second, Max: 60 * time.Second, diff --git a/libbeat/outputs/shipper/shipper.go b/libbeat/outputs/shipper/shipper.go index eceedff9cd7..6163b6fb8f9 100644 --- a/libbeat/outputs/shipper/shipper.go +++ b/libbeat/outputs/shipper/shipper.go @@ -20,7 +20,7 @@ package shipper import ( "context" "fmt" - "time" + "sync" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/outputs" @@ -39,16 +39,33 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) +type pendingBatch struct { + batch publisher.Batch + index uint64 + serverID string + droppedCount int +} + type shipper struct { log *logp.Logger observer outputs.Observer - conn *grpc.ClientConn - client sc.ProducerClient - timeout time.Duration + config Config + serverID string + + pending []pendingBatch + pendingMutex sync.Mutex + + conn *grpc.ClientConn + client sc.ProducerClient + clientMutex sync.Mutex + + backgroundCtx context.Context + backgroundCancel func() } func init() { @@ -68,25 +85,29 @@ func makeShipper( return outputs.Fail(err) } - s := outputs.WithBackoff(&shipper{ + s := &shipper{ log: logp.NewLogger("shipper"), observer: observer, config: config, - timeout: config.Timeout, - }, config.Backoff.Init, config.Backoff.Max) + } + + // for `Close` function to stop all the background work like acknowledgment loop + s.backgroundCtx, s.backgroundCancel = context.WithCancel(context.Background()) + + swb := outputs.WithBackoff(s, config.Backoff.Init, config.Backoff.Max) - return outputs.Success(config.BulkMaxSize, config.MaxRetries, s) + return outputs.Success(config.BulkMaxSize, config.MaxRetries, swb) } // Connect establishes connection to the shipper server and implements `outputs.Connectable`. -func (c *shipper) Connect() error { - tls, err := tlscommon.LoadTLSConfig(c.config.TLS) +func (s *shipper) Connect() error { + tls, err := tlscommon.LoadTLSConfig(s.config.TLS) if err != nil { return fmt.Errorf("invalid shipper TLS configuration: %w", err) } var creds credentials.TransportCredentials - if c.config.TLS != nil && c.config.TLS.Enabled != nil && *c.config.TLS.Enabled { + if s.config.TLS != nil && s.config.TLS.Enabled != nil && *s.config.TLS.Enabled { creds = credentials.NewTLS(tls.ToConfig()) } else { creds = insecure.NewCredentials() @@ -94,119 +115,213 @@ func (c *shipper) Connect() error { opts := []grpc.DialOption{ grpc.WithConnectParams(grpc.ConnectParams{ - MinConnectTimeout: c.config.Timeout, + MinConnectTimeout: s.config.Timeout, }), grpc.WithBlock(), grpc.WithTransportCredentials(creds), } - ctx, cancel := context.WithTimeout(context.Background(), c.config.Timeout) + ctx, cancel := context.WithTimeout(context.Background(), s.config.Timeout) defer cancel() - c.log.Debugf("trying to connect to %s...", c.config.Server) + s.log.Debugf("trying to connect to %s...", s.config.Server) - conn, err := grpc.DialContext(ctx, c.config.Server, opts...) + conn, err := grpc.DialContext(ctx, s.config.Server, opts...) if err != nil { return fmt.Errorf("shipper connection failed with: %w", err) } - c.log.Debugf("connect to %s established.", c.config.Server) - c.conn = conn - c.client = sc.NewProducerClient(conn) + s.conn = conn + s.clientMutex.Lock() + defer s.clientMutex.Unlock() + + s.client = sc.NewProducerClient(conn) + + // we don't need a timeout context here anymore, we use the + // `s.backgroundCtx` instead, it's going to be a long running client + ackCtx, ackCancel := context.WithCancel(s.backgroundCtx) + defer func() { + // in case we return an error before we start the `ackLoop` + // then we don't need this client anymore and must close the stream + if err != nil { + ackCancel() + } + }() + + indexClient, err := s.client.PersistedIndex(ackCtx, &messages.PersistedIndexRequest{ + PollingInterval: durationpb.New(s.config.AckPollingInterval), + }) + if err != nil { + return fmt.Errorf("failed to connect to the server: %w", err) + } + indexReply, err := indexClient.Recv() + if err != nil { + return fmt.Errorf("failed to fetch server information: %w", err) + } + s.serverID = indexReply.GetUuid() + + s.log.Debugf("connection to %s (%s) established.", s.config.Server, s.serverID) + + go func() { + defer ackCancel() + s.log.Debugf("starting acknowledgment loop with server %s", s.serverID) + // the loop returns only in case of error + err := s.ackLoop(s.backgroundCtx, indexClient) + s.log.Errorf("acknowledgment loop stopped: %s", err) + }() return nil } // Publish converts and sends a batch of events to the shipper server. // Also, implements `outputs.Client` -func (c *shipper) Publish(ctx context.Context, batch publisher.Batch) error { - if c.client == nil { +func (s *shipper) Publish(ctx context.Context, batch publisher.Batch) error { + if s.client == nil { return fmt.Errorf("connection is not established") } - st := c.observer + st := s.observer events := batch.Events() st.NewBatch(len(events)) - nonDroppedEvents := make([]publisher.Event, 0, len(events)) - convertedEvents := make([]*messages.Event, 0, len(events)) + toSend := make([]*messages.Event, 0, len(events)) - c.log.Debugf("converting %d events to protobuf...", len(events)) + s.log.Debugf("converting %d events to protobuf...", len(events)) - for i, e := range events { + droppedCount := 0 + for i, e := range events { converted, err := toShipperEvent(e) if err != nil { // conversion errors are not recoverable, so we have to drop the event completely - c.log.Errorf("%d/%d: %q, dropped", i+1, len(events), err) + s.log.Errorf("%d/%d: %q, dropped", i+1, len(events), err) + droppedCount++ continue } - convertedEvents = append(convertedEvents, converted) - nonDroppedEvents = append(nonDroppedEvents, e) + toSend = append(toSend, converted) } - droppedCount := len(events) - len(nonDroppedEvents) + convertedCount := len(toSend) st.Dropped(droppedCount) - c.log.Debugf("%d events converted to protobuf, %d dropped", len(nonDroppedEvents), droppedCount) + s.log.Debugf("%d events converted to protobuf, %d dropped", convertedCount, droppedCount) + + var lastAcceptedIndex uint64 - ctx, cancel := context.WithTimeout(ctx, c.timeout) + ctx, cancel := context.WithTimeout(ctx, s.config.Timeout) defer cancel() - resp, err := c.client.PublishEvents(ctx, &messages.PublishRequest{ - Events: convertedEvents, - }) - if status.Code(err) != codes.OK || resp == nil { - batch.Cancelled() // does not decrease the TTL - st.Cancelled(len(events)) // we cancel the whole batch not just non-dropped events - return fmt.Errorf("failed to publish the batch to the shipper, none of the %d events were accepted: %w", len(convertedEvents), err) - } + for len(toSend) > 0 { + publishReply, err := s.client.PublishEvents(ctx, &messages.PublishRequest{ + Uuid: s.serverID, + Events: toSend, + }) - // with a correct server implementation should never happen, this error is not recoverable - if int(resp.AcceptedCount) > len(nonDroppedEvents) { - return fmt.Errorf( - "server returned unexpected results, expected maximum accepted items %d, got %d", - len(nonDroppedEvents), - resp.AcceptedCount, - ) - } + if status.Code(err) != codes.OK { + batch.Cancelled() // does not decrease the TTL + st.Cancelled(len(events)) // we cancel the whole batch not just non-dropped events + return fmt.Errorf("failed to publish the batch to the shipper, none of the %d events were accepted: %w", len(toSend), err) + } - // the server is supposed to retain the order of the initial events in the response - // judging by the size of the result list we can determine what part of the initial - // list was accepted and we can send the rest of the list for a retry - retries := nonDroppedEvents[resp.AcceptedCount:] - if len(retries) == 0 { - batch.ACK() - st.Acked(len(nonDroppedEvents)) - c.log.Debugf("%d events have been accepted, %d dropped", len(nonDroppedEvents), droppedCount) - } else { - batch.RetryEvents(retries) // decreases TTL unless guaranteed delivery - st.Failed(len(retries)) - c.log.Debugf("%d events have been accepted, %d sent for retry, %d dropped", resp.AcceptedCount, len(retries), droppedCount) + // with a correct server implementation should never happen, this error is not recoverable + if int(publishReply.AcceptedCount) > len(toSend) { + return fmt.Errorf( + "server returned unexpected results, expected maximum accepted items %d, got %d", + len(toSend), + publishReply.AcceptedCount, + ) + } + toSend = toSend[publishReply.AcceptedCount:] + lastAcceptedIndex = publishReply.AcceptedIndex + s.log.Debugf("%d events have been accepted during a publish request", len(toSend)) } + s.log.Debugf("total of %d events have been accepted from batch, %d dropped", convertedCount, droppedCount) + + s.pendingMutex.Lock() + s.pending = append(s.pending, pendingBatch{ + batch: batch, + index: lastAcceptedIndex, + serverID: s.serverID, + droppedCount: droppedCount, + }) + s.pendingMutex.Unlock() + return nil } // Close closes the connection to the shipper server. // Also, implements `outputs.Client` -func (c *shipper) Close() error { - if c.client == nil { +func (s *shipper) Close() error { + if s.client == nil { return fmt.Errorf("connection is not established") } - err := c.conn.Close() - c.conn = nil - c.client = nil + s.backgroundCancel() + err := s.conn.Close() + s.conn = nil + s.client = nil + s.pending = nil return err } // String implements `outputs.Client` -func (c *shipper) String() string { +func (s *shipper) String() string { return "shipper" } +func (s *shipper) ackLoop(ctx context.Context, ackClient sc.Producer_PersistedIndexClient) error { + st := s.observer + + for { + select { + + case <-ctx.Done(): + return ctx.Err() + + default: + // this sends an update and unblocks only if the `PersistedIndex` value has changed + indexReply, err := ackClient.Recv() + if err != nil { + return fmt.Errorf("acknowledgment failed due to the connectivity error: %w", err) + } + + s.pendingMutex.Lock() + lastProcessed := 0 + for _, p := range s.pending { + if p.serverID != indexReply.Uuid { + s.log.Errorf("acknowledgment failed due to a connection to a different server %s, batch was accepted by %s", indexReply.Uuid, p.serverID) + p.batch.Cancelled() + st.Cancelled(len(p.batch.Events())) + lastProcessed++ + continue + } + + // if we met a batch that is ahead of the persisted index + // we stop iterating and wait for another update from the server. + // The latest pending batch has the max(AcceptedIndex). + if p.index > indexReply.PersistedIndex { + break + } + + p.batch.ACK() + ackedCount := len(p.batch.Events()) - p.droppedCount + st.Acked(ackedCount) + s.log.Debugf("%d events have been acknowledged, %d dropped", ackedCount, p.droppedCount) + lastProcessed++ + } + // so we don't perform any manipulation when the pending list is empty + // or none of the batches were acknowledged by this persisted index update + if lastProcessed != 0 { + copy(s.pending[0:], s.pending[lastProcessed:]) + s.pending = s.pending[lastProcessed:] + } + s.pendingMutex.Unlock() + } + } +} + func convertMapStr(m mapstr.M) (*messages.Value, error) { if m == nil { return helpers.NewNullValue(), nil diff --git a/libbeat/outputs/shipper/shipper_test.go b/libbeat/outputs/shipper/shipper_test.go index 08451006433..af028f886a7 100644 --- a/libbeat/outputs/shipper/shipper_test.go +++ b/libbeat/outputs/shipper/shipper_test.go @@ -22,10 +22,12 @@ import ( "errors" "fmt" "net" + "reflect" "strings" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -247,12 +249,13 @@ func TestPublish(t *testing.T) { } cases := []struct { - name string - events []beat.Event - expSignals []outest.BatchSignal - serverError error - expError string - qSize int + name string + events []beat.Event + expSignals []outest.BatchSignal + serverError error + expError string + qSize int + acceptedCount uint32 }{ { name: "sends a batch excluding dropped", @@ -269,11 +272,11 @@ func TestPublish(t *testing.T) { events: events, expSignals: []outest.BatchSignal{ { - Tag: outest.BatchRetryEvents, - Events: toPublisherEvents(events[2:]), + Tag: outest.BatchACK, }, }, - qSize: 1, + qSize: 2, + acceptedCount: 1, // we'll enforce 2 `PublishEvents` requests }, { name: "cancels the batch if server error", @@ -297,7 +300,7 @@ func TestPublish(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - addr, stop := runServer(t, tc.qSize, tc.serverError, "localhost:0") + addr, producer, stop := runServer(t, tc.qSize, tc.serverError, "localhost:0") defer stop() cfg, err := config.NewConfigFrom(map[string]interface{}{ @@ -305,37 +308,33 @@ func TestPublish(t *testing.T) { }) require.NoError(t, err) - group, err := makeShipper( - nil, - beat.Info{Beat: "libbeat", IndexPrefix: "testbeat"}, - outputs.NewNilObserver(), - cfg, - ) - require.NoError(t, err) - require.Len(t, group.Clients, 1) + client := createShipperClient(t, cfg) batch := outest.NewBatch(tc.events...) - err = group.Clients[0].(outputs.Connectable).Connect() - require.NoError(t, err) - - err = group.Clients[0].Publish(ctx, batch) + err = client.Publish(ctx, batch) if tc.expError != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.expError) } else { require.NoError(t, err) + producer.Persist(uint64(tc.qSize)) // always persisted all published events } + assert.Eventually(t, func() bool { + // there is a background routine that checks acknowledgments, + // it should eventually change the status of the batch + return reflect.DeepEqual(tc.expSignals, batch.Signals) + }, 100*time.Millisecond, 10*time.Millisecond) require.Equal(t, tc.expSignals, batch.Signals) }) } - t.Run("cancel the batch when the server is not available", func(t *testing.T) { + t.Run("cancels the batch when a different server responds", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - addr, stop := runServer(t, 5, nil, "localhost:0") + addr, _, stop := runServer(t, 5, nil, "localhost:0") defer stop() cfg, err := config.NewConfigFrom(map[string]interface{}{ @@ -348,64 +347,84 @@ func TestPublish(t *testing.T) { }) require.NoError(t, err) - group, err := makeShipper( - nil, - beat.Info{Beat: "libbeat", IndexPrefix: "testbeat"}, - outputs.NewNilObserver(), - cfg, - ) + client := createShipperClient(t, cfg) + + // Should accept the batch and put it to the pending list + batch := outest.NewBatch(events...) + err = client.Publish(ctx, batch) require.NoError(t, err) - require.Len(t, group.Clients, 1) - client := group.Clients[0].(outputs.NetworkClient) + // Replace the server (would change the ID) + stop() + _, _, stop = runServer(t, 5, nil, addr) + defer stop() err = client.Connect() require.NoError(t, err) - // Should successfully publish with the server running - batch := outest.NewBatch(events...) - err = client.Publish(ctx, batch) - require.NoError(t, err) expSignals := []outest.BatchSignal{ { - Tag: outest.BatchACK, + Tag: outest.BatchCancelled, }, } + assert.Eventually(t, func() bool { + // there is a background routine that checks acknowledgments, + // it should eventually cancel the batch because the IDs don't match + return reflect.DeepEqual(expSignals, batch.Signals) + }, 100*time.Millisecond, 10*time.Millisecond) require.Equal(t, expSignals, batch.Signals) + }) - stop() // now stop the server and try sending again + t.Run("acks multiple batches", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() - batch = outest.NewBatch(events...) // resetting the batch signals - err = client.Publish(ctx, batch) - require.Error(t, err) - require.Contains(t, err.Error(), "failed to publish the batch to the shipper, none of the 2 events were accepted") - expSignals = []outest.BatchSignal{ - { - Tag: outest.BatchCancelled, // "cancelled" means there will be a retry without decreasing the TTL + addr, producer, stop := runServer(t, 6, nil, "localhost:0") + defer stop() + + cfg, err := config.NewConfigFrom(map[string]interface{}{ + "server": addr, + "timeout": 5, // 5 sec + "backoff": map[string]interface{}{ + "init": "10ms", + "max": "5s", }, - } - require.Equal(t, expSignals, batch.Signals) - client.Close() + }) + require.NoError(t, err) - // Start the server again - _, stop = runServer(t, 5, nil, addr) - defer stop() + client := createShipperClient(t, cfg) + + // Should accept the batch and put it to the pending list + batch1 := outest.NewBatch(events...) + err = client.Publish(ctx, batch1) + require.NoError(t, err) + + batch2 := outest.NewBatch(events...) + err = client.Publish(ctx, batch2) + require.NoError(t, err) - batch = outest.NewBatch(events...) // resetting the signals - expSignals = []outest.BatchSignal{ + batch3 := outest.NewBatch(events...) + err = client.Publish(ctx, batch3) + require.NoError(t, err) + + expSignals := []outest.BatchSignal{ { Tag: outest.BatchACK, }, } - // The backoff wrapper should take care of the errors and - // retries while the server is still starting - err = client.Connect() - require.NoError(t, err) - - err = client.Publish(ctx, batch) - require.NoError(t, err) - require.Equal(t, expSignals, batch.Signals) + producer.Persist(6) // 2 events per batch, 3 batches + + assert.Eventually(t, func() bool { + // there is a background routine that checks acknowledgments, + // it should eventually send expected signals + return reflect.DeepEqual(expSignals, batch1.Signals) && + reflect.DeepEqual(expSignals, batch2.Signals) && + reflect.DeepEqual(expSignals, batch3.Signals) + }, 100*time.Millisecond, 10*time.Millisecond) + require.Equal(t, expSignals, batch1.Signals, "batch1") + require.Equal(t, expSignals, batch2.Signals, "batch2") + require.Equal(t, expSignals, batch3.Signals, "batch3") }) } @@ -464,7 +483,7 @@ func BenchmarkToShipperEvent(b *testing.B) { // `err` is a preset error that the server will serve to the client // `listenAddr` is the address for the server to listen // returns `actualAddr` where the listener actually is and the `stop` function to stop the server -func runServer(t *testing.T, qSize int, err error, listenAddr string) (actualAddr string, stop func()) { +func runServer(t *testing.T, qSize int, err error, listenAddr string) (actualAddr string, mock *api.ProducerMock, stop func()) { producer := api.NewProducerMock(qSize) producer.Error = err grpcServer := grpc.NewServer() @@ -482,7 +501,25 @@ func runServer(t *testing.T, qSize int, err error, listenAddr string) (actualAdd listener.Close() } - return actualAddr, stop + return actualAddr, producer, stop +} + +func createShipperClient(t *testing.T, cfg *config.C) outputs.NetworkClient { + group, err := makeShipper( + nil, + beat.Info{Beat: "libbeat", IndexPrefix: "testbeat"}, + outputs.NewNilObserver(), + cfg, + ) + require.NoError(t, err) + require.Len(t, group.Clients, 1) + + client := group.Clients[0].(outputs.NetworkClient) + + err = client.Connect() + require.NoError(t, err) + + return client } func protoStruct(t *testing.T, values map[string]interface{}) *messages.Struct { @@ -502,11 +539,3 @@ func requireEqualProto(t *testing.T, expected, actual proto.Message) { fmt.Sprintf("These two protobuf messages are not equal:\nexpected: %v\nactual: %v", expected, actual), ) } - -func toPublisherEvents(events []beat.Event) []publisher.Event { - converted := make([]publisher.Event, 0, len(events)) - for _, e := range events { - converted = append(converted, publisher.Event{Content: e}) - } - return converted -} diff --git a/libbeat/processors/util/netinfo.go b/libbeat/processors/util/netinfo.go index 3112152db19..d4e694e5fa4 100644 --- a/libbeat/processors/util/netinfo.go +++ b/libbeat/processors/util/netinfo.go @@ -41,10 +41,8 @@ func GetNetInfo() (ipList []string, hwList []string, err error) { continue } - hw := i.HardwareAddr.String() - // Skip empty hardware addresses - if hw != "" { - hwList = append(hwList, hw) + if len(i.HardwareAddr) != 0 { + hwList = append(hwList, formatHardwareAddr(i.HardwareAddr)) } addrs, err := i.Addrs() @@ -67,6 +65,19 @@ func GetNetInfo() (ipList []string, hwList []string, err error) { return ipList, unique(hwList), errs.Err() } +// formatHardwareAddr formats hardware addresses according to the ECS spec. +func formatHardwareAddr(addr net.HardwareAddr) string { + buf := make([]byte, 0, len(addr)*3-1) + for _, b := range addr { + if len(buf) != 0 { + buf = append(buf, '-') + } + const hexDigit = "0123456789ABCDEF" + buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf]) + } + return string(buf) +} + // unique returns addrs lexically sorted and with repeated elements // omitted. func unique(addrs []string) []string { diff --git a/libbeat/processors/util/netinfo_test.go b/libbeat/processors/util/netinfo_test.go index 186f00debf3..17c47f965a6 100644 --- a/libbeat/processors/util/netinfo_test.go +++ b/libbeat/processors/util/netinfo_test.go @@ -18,13 +18,16 @@ package util import ( + "net" "reflect" + "regexp" "sort" + "strings" "testing" ) func TestUnique(t *testing.T) { - var tests = [][]string{ + tests := [][]string{ {}, {"a"}, {"a", "a"}, @@ -54,3 +57,31 @@ func TestUnique(t *testing.T) { } } } + +func TestFormatHardwareAddr(t *testing.T) { + tests := []string{ + "00:00:5e:00:53:01", + "02:00:5e:10:00:00:00:01", + "00:00:00:00:fe:80:00:00:00:00:00:00:02:00:5e:10:00:00:00:01", + "00-00-5e-00-53-01", + "02-00-5e-10-00-00-00-01", + "00-00-00-00-fe-80-00-00-00-00-00-00-02-00-5e-10-00-00-00-01", + "0000.5e00.5301", + "0200.5e10.0000.0001", + "0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001", + } + + spec := regexp.MustCompile(`[0-9A-F]{2}(?:[0-9A-F]{2})*`) + for _, test := range tests { + addr, err := net.ParseMAC(test) + if err != nil { + t.Errorf("failed to parse test case %q", test) + continue + } + got := formatHardwareAddr(addr) + want := strings.ToUpper(strings.ReplaceAll(addr.String(), ":", "-")) + if got != want || !spec.MatchString(got) { + t.Errorf("unexpected format for %q: got:%q want:%q", test, got, want) + } + } +} diff --git a/libbeat/publisher/pipeline/client.go b/libbeat/publisher/pipeline/client.go index 205245f7195..9ccf7d170f9 100644 --- a/libbeat/publisher/pipeline/client.go +++ b/libbeat/publisher/pipeline/client.go @@ -125,9 +125,9 @@ func (c *client) publish(e beat.Event) { var published bool if c.canDrop { - published = c.producer.TryPublish(pubEvent) + _, published = c.producer.TryPublish(pubEvent) } else { - published = c.producer.Publish(pubEvent) + _, published = c.producer.Publish(pubEvent) } if published { diff --git a/libbeat/publisher/pipeline/client_test.go b/libbeat/publisher/pipeline/client_test.go index a0ea83a890d..a23763841f5 100644 --- a/libbeat/publisher/pipeline/client_test.go +++ b/libbeat/publisher/pipeline/client_test.go @@ -187,7 +187,6 @@ func TestClientWaitClose(t *testing.T) { defer client.Close() // Send an event which gets acknowledged immediately. - client.Publish(beat.Event{}) output := newMockClient(func(batch publisher.Batch) error { batch.ACK() return nil @@ -196,6 +195,8 @@ func TestClientWaitClose(t *testing.T) { pipeline.output.Set(outputs.Group{Clients: []outputs.Client{output}}) defer pipeline.output.Set(outputs.Group{}) + client.Publish(beat.Event{}) + closed := make(chan struct{}) go func() { defer close(closed) diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index f28d3eb623b..27280182a2f 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -266,9 +266,10 @@ func (p *Pipeline) ConnectWith(cfg beat.ClientConfig) (beat.Client, error) { producerCfg := queue.ProducerConfig{} if reportEvents || cfg.Events != nil { - producerCfg.OnDrop = func(event beat.Event) { + producerCfg.OnDrop = func(event interface{}) { + publisherEvent, _ := event.(publisher.Event) if cfg.Events != nil { - cfg.Events.DroppedOnPublish(event) + cfg.Events.DroppedOnPublish(publisherEvent.Content) } if reportEvents { p.waitCloseGroup.Add(-1) diff --git a/libbeat/publisher/pipeline/pipeline_test.go b/libbeat/publisher/pipeline/pipeline_test.go index 57cdbd635ea..cd6703697f4 100644 --- a/libbeat/publisher/pipeline/pipeline_test.go +++ b/libbeat/publisher/pipeline/pipeline_test.go @@ -32,7 +32,7 @@ type testQueue struct { } type testProducer struct { - publish func(try bool, event interface{}) bool + publish func(try bool, event interface{}) (queue.EntryID, bool) cancel func() int } @@ -68,18 +68,18 @@ func (q *testQueue) Get(sz int) (queue.Batch, error) { return nil, nil } -func (p *testProducer) Publish(event interface{}) bool { +func (p *testProducer) Publish(event interface{}) (queue.EntryID, bool) { if p.publish != nil { return p.publish(false, event) } - return false + return 0, false } -func (p *testProducer) TryPublish(event interface{}) bool { +func (p *testProducer) TryPublish(event interface{}) (queue.EntryID, bool) { if p.publish != nil { return p.publish(true, event) } - return false + return 0, false } func (p *testProducer) Cancel() int { @@ -114,7 +114,7 @@ func makeTestQueue() queue.Queue { var producer *testProducer p := blockingProducer(cfg) producer = &testProducer{ - publish: func(try bool, event interface{}) bool { + publish: func(try bool, event interface{}) (queue.EntryID, bool) { if try { return p.TryPublish(event) } @@ -146,10 +146,10 @@ func blockingProducer(_ queue.ProducerConfig) queue.Producer { waiting := atomic.MakeInt(0) return &testProducer{ - publish: func(_ bool, _ interface{}) bool { + publish: func(_ bool, _ interface{}) (queue.EntryID, bool) { waiting.Inc() <-sig - return false + return 0, false }, cancel: func() int { diff --git a/libbeat/publisher/pipeline/ttl_batch.go b/libbeat/publisher/pipeline/ttl_batch.go index 39155442ba0..0bf69416908 100644 --- a/libbeat/publisher/pipeline/ttl_batch.go +++ b/libbeat/publisher/pipeline/ttl_batch.go @@ -51,7 +51,7 @@ func newBatch(retryer retryer, original queue.Batch, ttl int) *ttlBatch { count := original.Count() events := make([]publisher.Event, 0, count) for i := 0; i < count; i++ { - event, ok := original.Event(i).(publisher.Event) + event, ok := original.Entry(i).(publisher.Event) if ok { // In Beats this conversion will always succeed because only // publisher.Event objects are inserted into the queue, but diff --git a/libbeat/publisher/queue/diskqueue/benchmark_test.go b/libbeat/publisher/queue/diskqueue/benchmark_test.go index 928cf4806cd..bf2a2ab1e13 100644 --- a/libbeat/publisher/queue/diskqueue/benchmark_test.go +++ b/libbeat/publisher/queue/diskqueue/benchmark_test.go @@ -132,7 +132,7 @@ func publishEvents(p queue.Producer, num int, protobuf bool) { } else { e = makePublisherEvent() } - ok := p.Publish(e) + _, ok := p.Publish(e) if !ok { panic("didn't publish") } diff --git a/libbeat/publisher/queue/diskqueue/consumer.go b/libbeat/publisher/queue/diskqueue/consumer.go index 1e7d6361383..5da9ac7fd7b 100644 --- a/libbeat/publisher/queue/diskqueue/consumer.go +++ b/libbeat/publisher/queue/diskqueue/consumer.go @@ -86,10 +86,14 @@ func (batch *diskQueueBatch) Count() int { return len(batch.frames) } -func (batch *diskQueueBatch) Event(i int) interface{} { +func (batch *diskQueueBatch) Entry(i int) interface{} { return batch.frames[i].event } +func (batch *diskQueueBatch) ID(i int) queue.EntryID { + return 0 +} + func (batch *diskQueueBatch) Done() { batch.queue.acks.addFrames(batch.frames) } diff --git a/libbeat/publisher/queue/diskqueue/producer.go b/libbeat/publisher/queue/diskqueue/producer.go index b7ae0be2772..7471c2b4701 100644 --- a/libbeat/publisher/queue/diskqueue/producer.go +++ b/libbeat/publisher/queue/diskqueue/producer.go @@ -49,12 +49,12 @@ type producerWriteRequest struct { // diskQueueProducer implementation of the queue.Producer interface // -func (producer *diskQueueProducer) Publish(event interface{}) bool { - return producer.publish(event, true) +func (producer *diskQueueProducer) Publish(event interface{}) (queue.EntryID, bool) { + return 0, producer.publish(event, true) } -func (producer *diskQueueProducer) TryPublish(event interface{}) bool { - return producer.publish(event, false) +func (producer *diskQueueProducer) TryPublish(event interface{}) (queue.EntryID, bool) { + return 0, producer.publish(event, false) } func (producer *diskQueueProducer) publish( diff --git a/libbeat/publisher/queue/memqueue/ackloop.go b/libbeat/publisher/queue/memqueue/ackloop.go index 6d467e287a9..1c6cc43d265 100644 --- a/libbeat/publisher/queue/memqueue/ackloop.go +++ b/libbeat/publisher/queue/memqueue/ackloop.go @@ -29,41 +29,27 @@ type ackLoop struct { // used to maintain sequencing of event acknowledgements. ackChans chanList - totalACK uint64 - processACK func(chanList, int) } func (l *ackLoop) run() { - var ( - // Buffer up event counter in ackCount. If ackCount > 0, acks will be set to - // the broker.acks channel for sending the ACKs while potentially receiving - // new batches from the broker event loop. - // This concurrent bidirectionally communication pattern requiring 'select' - // ensures we can not have any deadlock between the event loop and the ack - // loop, as the ack loop will not block on any channel - ackCount int - ackChan chan int - ) - for { nextBatchChan := l.ackChans.nextBatchChannel() select { case <-l.broker.done: + // The queue is shutting down. return - case ackChan <- ackCount: - ackChan, ackCount = nil, 0 - case chanList := <-l.broker.scheduledACKs: + // A new batch has been generated, add its ACK channel to the end of + // the pending list. l.ackChans.concat(&chanList) case <-nextBatchChan: - ackCount += l.handleBatchSig() - if ackCount > 0 { - ackChan = l.broker.ackChan - } + // The oldest outstanding batch has been acknowledged, advance our + // position as much as we can. + l.handleBatchSig() } } } @@ -94,7 +80,6 @@ func (l *ackLoop) handleBatchSig() int { // return final ACK to EventLoop, in order to clean up internal buffer l.broker.logger.Debug("ackloop: return ack to broker loop:", count) - l.totalACK += uint64(count) l.broker.logger.Debug("ackloop: done send ack") return count } diff --git a/libbeat/publisher/queue/memqueue/batchbuf.go b/libbeat/publisher/queue/memqueue/batchbuf.go index 53c625688f3..87c3a1052f3 100644 --- a/libbeat/publisher/queue/memqueue/batchbuf.go +++ b/libbeat/publisher/queue/memqueue/batchbuf.go @@ -17,11 +17,6 @@ package memqueue -type queueEntry struct { - event interface{} - client clientState -} - type batchBuffer struct { next *batchBuffer flushed bool @@ -34,20 +29,20 @@ func newBatchBuffer(sz int) *batchBuffer { return b } -func (b *batchBuffer) add(event interface{}, st clientState) { - b.entries = append(b.entries, queueEntry{event, st}) +func (b *batchBuffer) add(entry queueEntry) { + b.entries = append(b.entries, entry) } func (b *batchBuffer) length() int { return len(b.entries) } -func (b *batchBuffer) cancel(st *produceState) int { +func (b *batchBuffer) cancel(producer *ackProducer) int { entries := b.entries[:0] removedCount := 0 for _, entry := range b.entries { - if entry.client.state == st { + if entry.producer == producer { removedCount++ continue } diff --git a/libbeat/publisher/queue/memqueue/broker.go b/libbeat/publisher/queue/memqueue/broker.go index ed4c91565bb..52fe8e62c68 100644 --- a/libbeat/publisher/queue/memqueue/broker.go +++ b/libbeat/publisher/queue/memqueue/broker.go @@ -57,11 +57,6 @@ type broker struct { /////////////////////////// // internal channels - // When ackLoop receives events ACKs from a consumer, it sends the number - // of ACKed events to ackChan to notify the event loop that those - // events can be removed from the queue. - ackChan chan int - // When events are sent to consumers, the ACK channels for their batches // are collected into chanLists and sent to scheduledACKs. // These are then read by ackLoop and concatenated to its internal @@ -95,6 +90,14 @@ type Settings struct { InputQueueSize int } +type queueEntry struct { + event interface{} + id queue.EntryID + + producer *ackProducer + producerID producerID // The order of this entry within its producer +} + type batch struct { queue *broker entries []queueEntry @@ -153,7 +156,7 @@ func create( func NewQueue( logger *logp.Logger, settings Settings, -) queue.Queue { +) *broker { var ( sz = settings.Events minEvents = settings.FlushMinEvents @@ -187,7 +190,6 @@ func NewQueue( cancelChan: make(chan producerCancelRequest, 5), // internal broker and ACK handler channels - ackChan: make(chan int), scheduledACKs: make(chan chanList), ackListener: settings.ACKListener, @@ -271,6 +273,7 @@ func (b *broker) Metrics() (queue.Metrics, error) { EventCount: opt.UintWith(uint64(resp.currentQueueSize)), EventLimit: opt.UintWith(uint64(b.bufSize)), UnackedConsumedEvents: opt.UintWith(uint64(resp.occupiedRead)), + OldestEntryID: resp.oldestEntryID, }, nil } @@ -380,10 +383,14 @@ func (b *batch) Count() int { return len(b.entries) } -func (b *batch) Event(i int) interface{} { +func (b *batch) Entry(i int) interface{} { return b.entries[i].event } +func (b *batch) ID(i int) queue.EntryID { + return b.entries[i].id +} + func (b *batch) Done() { b.doneChan <- batchDoneMsg{} } diff --git a/libbeat/publisher/queue/memqueue/eventloop.go b/libbeat/publisher/queue/memqueue/eventloop.go index b6dc4af470d..ccb50565365 100644 --- a/libbeat/publisher/queue/memqueue/eventloop.go +++ b/libbeat/publisher/queue/memqueue/eventloop.go @@ -18,51 +18,67 @@ package memqueue import ( - "fmt" - "math" "time" - "github.com/elastic/beats/v7/libbeat/publisher" + "github.com/elastic/beats/v7/libbeat/publisher/queue" "github.com/elastic/elastic-agent-libs/logp" ) // directEventLoop implements the broker main event loop. It buffers events, // but tries to forward events as early as possible. type directEventLoop struct { - broker *broker - buf ringBuffer + broker *broker + buf ringBuffer + deleteChan chan int // pendingACKs aggregates a list of ACK channels for batches that have been sent // to consumers, which is then sent to the broker's scheduledACKs channel. pendingACKs chanList + + nextEntryID queue.EntryID } // bufferingEventLoop implements the broker main event loop. // Events in the buffer are forwarded to consumers only if the buffer is full or on flush timeout. type bufferingEventLoop struct { - broker *broker + broker *broker + deleteChan chan int + + // The current buffer that incoming events are appended to. When it gets + // full enough, or enough time has passed, it is added to flushList. + // Events will still be added to buf even after it is in flushList, until + // either it reaches minEvents or a consumer requests it. + buf *batchBuffer - buf *batchBuffer + // flushList is the list of buffers that are ready to be sent to consumers. flushList flushList + // pendingACKs aggregates a list of ACK channels for batches that have been sent + // to consumers, which is then sent to the broker's scheduledACKs channel. + pendingACKs chanList + // The number of events currently waiting in the queue, including // those that have not yet been acked. eventCount int - // The number of events that have been read by a consumer but not yet acked - unackedEventCount int + // The next entry ID that will be read by a consumer, and the next + // entry ID that has been consumed and is waiting for acknowledgment. + // We need to track these here because bufferingEventLoop discards + // its event buffers when they are sent to consumers, so we can't + // look directly at the event itself to get the current id like we + // do in the unbuffered loop. + nextConsumedID queue.EntryID + nextACKedID queue.EntryID minEvents int maxEvents int flushTimeout time.Duration - // pendingACKs aggregates a list of ACK channels for batches that have been sent - // to consumers, which is then sent to the broker's scheduledACKs channel. - pendingACKs chanList - // buffer flush timer state timer *time.Timer idleC <-chan time.Time + + nextEntryID queue.EntryID } type flushList struct { @@ -73,7 +89,8 @@ type flushList struct { func newDirectEventLoop(b *broker, size int) *directEventLoop { l := &directEventLoop{ - broker: b, + broker: b, + deleteChan: make(chan int), } l.buf.init(b.logger, size) @@ -114,8 +131,7 @@ func (l *directEventLoop) run() { case req := <-pushChan: // producer pushing new event l.insert(&req) - case count := <-l.broker.ackChan: - // Events have been ACKed, remove them from the internal buffer. + case count := <-l.deleteChan: l.buf.removeEntries(count) case req := <-l.broker.cancelChan: // producer cancelling active events @@ -136,23 +152,34 @@ func (l *directEventLoop) run() { } func (l *directEventLoop) handleMetricsRequest(req *metricsRequest) { - req.responseChan <- memQueueMetrics{currentQueueSize: l.buf.Items(), occupiedRead: l.buf.reserved} + // If the queue is empty, we report the "oldest" ID as the next + // one that will be assigned. Otherwise, we report the ID attached + // to the oldest queueEntry. + oldestEntryID := l.nextEntryID + if oldestEntry := l.buf.OldestEntry(); oldestEntry != nil { + oldestEntryID = oldestEntry.id + } + + req.responseChan <- memQueueMetrics{ + currentQueueSize: l.buf.Items(), + occupiedRead: l.buf.reserved, + oldestEntryID: oldestEntryID, + } } -// Returns true if the queue is full after handling the insertion request. func (l *directEventLoop) insert(req *pushRequest) { log := l.broker.logger - st := req.state - if st == nil { - l.buf.insert(req.event, clientState{}) - } else if st.cancelled { + if req.producer != nil && req.producer.state.cancelled { reportCancelledState(log, req) } else { - l.buf.insert(req.event, clientState{ - seq: req.seq, - state: st, - }) + req.resp <- l.nextEntryID + l.buf.insert(queueEntry{ + event: req.event, + id: l.nextEntryID, + producer: req.producer, + producerID: req.producerID}) + l.nextEntryID++ } } @@ -162,9 +189,9 @@ func (l *directEventLoop) handleCancel(req *producerCancelRequest) { var removed int - if st := req.state; st != nil { - st.cancelled = true - removed = l.buf.cancel(st) + if producer := req.producer; producer != nil { + producer.state.cancelled = true + removed = l.buf.cancel(producer) } // signal cancel request being finished @@ -203,55 +230,46 @@ func (l *directEventLoop) processACK(lst chanList, N int) { entries := l.buf.entries firstIndex := lst.front().start - // Position the index at the end of the block of ACKed events - idx := (firstIndex + N - 1) % len(entries) - total := 0 + // We want to acknowledge N events starting at position firstIndex + // in the entries array. + // We iterate over the events from last to first, so we encounter the + // highest producer IDs first and can skip subsequent callbacks to the + // same producer. + producerCallbacks := []func(){} for i := N - 1; i >= 0; i-- { - if idx < 0 { - idx = len(entries) - 1 - } - - client := &entries[idx].client - log.Debugf("try ack index: (idx=%v, i=%v, seq=%v)\n", idx, i, client.seq) - - idx-- - if client.state == nil { - log.Debug("no state set") + // idx is the index in entries of the i-th event after firstIndex, wrapping + // around the end of the array. + idx := (firstIndex + i) % len(entries) + entry := &entries[idx] + + producer := entry.producer + + // Set the producer in the entires array to nil to mark it as visited; a nil + // producer indicates that an entry requires no more ack processing (either + // because it has already been ACKed, or because its producer does not listen to ACKs). + entry.producer = nil + if producer == nil || entry.producerID <= producer.state.lastACK { + // This has a lower index than the previous ACK for this producer, + // so it was covered in the previous call and we can skip it. continue } + // This update is safe because lastACK is only used from the event loop. + count := int(entry.producerID - producer.state.lastACK) + producer.state.lastACK = entry.producerID - count := (client.seq - client.state.lastACK) - if count == 0 || count > math.MaxUint32/2 { - // seq number comparison did underflow. This happens only if st.seq has - // already been acknowledged - // log.Debug("seq number already acked: ", st.seq) - - client.state = nil - continue - } - - log.Debugf("broker ACK events: count=%v, start-seq=%v, end-seq=%v\n", - count, - client.state.lastACK+1, - client.seq, - ) - - total += int(count) - if total > N { - panic(fmt.Sprintf("Too many events acked (expected=%v, total=%v)", - N, total, - )) - } - client.state.cb(int(count)) - client.state.lastACK = client.seq - client.state = nil + producerCallbacks = append(producerCallbacks, func() { producer.state.cb(count) }) + } + l.deleteChan <- N + for _, f := range producerCallbacks { + f() } } func newBufferingEventLoop(b *broker, size int, minEvents int, flushTimeout time.Duration) *bufferingEventLoop { l := &bufferingEventLoop{ broker: b, + deleteChan: make(chan int), maxEvents: size, minEvents: minEvents, flushTimeout: flushTimeout, @@ -306,8 +324,8 @@ func (l *bufferingEventLoop) run() { case schedACKs <- l.pendingACKs: l.pendingACKs = chanList{} - case count := <-l.broker.ackChan: - l.handleACK(count) + case count := <-l.deleteChan: + l.handleDelete(count) case req := <-l.broker.metricChan: // broker asking for queue metrics l.handleMetricsRequest(&req) @@ -323,11 +341,19 @@ func (l *bufferingEventLoop) run() { } func (l *bufferingEventLoop) handleMetricsRequest(req *metricsRequest) { - req.responseChan <- memQueueMetrics{currentQueueSize: l.eventCount, occupiedRead: l.unackedEventCount} + req.responseChan <- memQueueMetrics{ + currentQueueSize: l.eventCount, + occupiedRead: int(l.nextConsumedID - l.nextACKedID), + oldestEntryID: l.nextACKedID, + } } func (l *bufferingEventLoop) handleInsert(req *pushRequest) { - if l.insert(req) { + if l.insert(req, l.nextEntryID) { + // Send back the new event id. + req.resp <- l.nextEntryID + + l.nextEntryID++ l.eventCount++ L := l.buf.length() @@ -339,45 +365,39 @@ func (l *bufferingEventLoop) handleInsert(req *pushRequest) { l.flushBuffer() l.buf = newBatchBuffer(l.minEvents) } - } else { - if L >= l.minEvents { - l.buf = newBatchBuffer(l.minEvents) - } + } else if L >= l.minEvents { + l.buf = newBatchBuffer(l.minEvents) } } } -func (l *bufferingEventLoop) insert(req *pushRequest) bool { - if req.state == nil { - l.buf.add(req.event, clientState{}) - return true - } - - st := req.state - if st.cancelled { +func (l *bufferingEventLoop) insert(req *pushRequest, id queue.EntryID) bool { + if req.producer != nil && req.producer.state.cancelled { reportCancelledState(l.broker.logger, req) return false } - l.buf.add(req.event, clientState{ - seq: req.seq, - state: st, + l.buf.add(queueEntry{ + event: req.event, + id: id, + producer: req.producer, + producerID: req.producerID, }) return true } func (l *bufferingEventLoop) handleCancel(req *producerCancelRequest) { removed := 0 - if st := req.state; st != nil { + if producer := req.producer; producer != nil { // remove from actively flushed buffers for buf := l.flushList.head; buf != nil; buf = buf.next { - removed += buf.cancel(st) + removed += buf.cancel(producer) } if !l.buf.flushed { - removed += l.buf.cancel(st) + removed += l.buf.cancel(producer) } - st.cancelled = true + producer.state.cancelled = true } if req.resp != nil { @@ -425,15 +445,15 @@ func (l *bufferingEventLoop) handleGetRequest(req *getRequest) { req.responseChan <- getResponse{acker.doneChan, entries} l.pendingACKs.append(acker) - l.unackedEventCount += len(entries) + l.nextConsumedID += queue.EntryID(len(entries)) buf.entries = buf.entries[count:] if buf.length() == 0 { l.advanceFlushList() } } -func (l *bufferingEventLoop) handleACK(count int) { - l.unackedEventCount -= count +func (l *bufferingEventLoop) handleDelete(count int) { + l.nextACKedID += queue.EntryID(count) l.eventCount -= count } @@ -462,57 +482,48 @@ func (l *bufferingEventLoop) advanceFlushList() { func (l *bufferingEventLoop) flushBuffer() { l.buf.flushed = true - - if l.buf.length() == 0 { - panic("flushing empty buffer") - } - l.flushList.add(l.buf) } +// Called by ackLoop. This function exists to decouple the work of collecting +// and running producer callbacks from logical deletion of the events, so +// input callbacks can't block the main queue goroutine. func (l *bufferingEventLoop) processACK(lst chanList, N int) { - log := l.broker.logger - - total := 0 + ackCallbacks := []func(){} + // First we traverse the entries we're about to remove, collecting any callbacks + // we need to run. lst.reverse() for !lst.empty() { current := lst.pop() entries := current.entries + // Traverse entries from last to first, so we can acknowledge the most recent + // ones first and skip subsequent producer callbacks. for i := len(entries) - 1; i >= 0; i-- { - st := &entries[i].client - if st.state == nil { + entry := &entries[i] + if entry.producer == nil { continue } - count := st.seq - st.state.lastACK - if count == 0 || count > math.MaxUint32/2 { - // seq number comparison did underflow. This happens only if st.seq has - // already been acknowledged - // log.Debug("seq number already acked: ", st.seq) - - st.state = nil + if entry.producerID <= entry.producer.state.lastACK { + // This index was already acknowledged on a previous iteration, skip. + entry.producer = nil continue } - - log.Debugf("broker ACK events: count=%v, start-seq=%v, end-seq=%v\n", - count, - st.state.lastACK+1, - st.seq, - ) - - total += int(count) - if total > N { - panic(fmt.Sprintf("Too many events acked (expected=%v, total=%v)", - N, total, - )) - } - - st.state.cb(int(count)) - st.state.lastACK = st.seq - st.state = nil + producerState := entry.producer.state + count := int(entry.producerID - producerState.lastACK) + ackCallbacks = append(ackCallbacks, func() { producerState.cb(count) }) + entry.producer.state.lastACK = entry.producerID + entry.producer = nil } } + // Signal the queue to delete the events + l.deleteChan <- N + + // The events have been removed; notify their listeners. + for _, f := range ackCallbacks { + f() + } } func (l *flushList) pop() { @@ -542,15 +553,8 @@ func (l *flushList) add(b *batchBuffer) { } func reportCancelledState(log *logp.Logger, req *pushRequest) { - log.Debugf("cancelled producer - ignore event: %v\t%v\t%p", req.event, req.seq, req.state) - // do not add waiting events if producer did send cancel signal - - st := req.state - if cb := st.dropCB; cb != nil { - if event, ok := req.event.(publisher.Event); ok { - cb(event.Content) - } + if cb := req.producer.state.dropCB; cb != nil { + cb(req.event) } - } diff --git a/libbeat/publisher/queue/memqueue/internal_api.go b/libbeat/publisher/queue/memqueue/internal_api.go index c924afc785c..07485af99b4 100644 --- a/libbeat/publisher/queue/memqueue/internal_api.go +++ b/libbeat/publisher/queue/memqueue/internal_api.go @@ -17,17 +17,26 @@ package memqueue +import "github.com/elastic/beats/v7/libbeat/publisher/queue" + // producer -> broker API type pushRequest struct { event interface{} - seq uint32 - state *produceState + + // The producer that generated this event, or nil if this producer does + // not require ack callbacks. + producer *ackProducer + + // The index of the event in this producer only. Used to condense + // multiple acknowledgments for a producer to a single callback call. + producerID producerID + resp chan queue.EntryID } type producerCancelRequest struct { - state *produceState - resp chan producerCancelResponse + producer *ackProducer + resp chan producerCancelResponse } type producerCancelResponse struct { @@ -60,4 +69,6 @@ type memQueueMetrics struct { currentQueueSize int // the number of items that have been read by a consumer but not yet ack'ed occupiedRead int + + oldestEntryID queue.EntryID } diff --git a/libbeat/publisher/queue/memqueue/produce.go b/libbeat/publisher/queue/memqueue/produce.go index dd87c481427..11a792af576 100644 --- a/libbeat/publisher/queue/memqueue/produce.go +++ b/libbeat/publisher/queue/memqueue/produce.go @@ -18,7 +18,6 @@ package memqueue import ( - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/publisher/queue" "github.com/elastic/elastic-agent-libs/logp" ) @@ -29,11 +28,11 @@ type forgetfulProducer struct { } type ackProducer struct { - broker *broker - dropOnCancel bool - seq uint32 - state produceState - openState openState + broker *broker + dropOnCancel bool + producedCount uint64 + state produceState + openState openState } type openState struct { @@ -42,16 +41,22 @@ type openState struct { events chan pushRequest } +// producerID stores the order of events within a single producer, so multiple +// event acknowledgement callbacks can be coalesced into a single call. +// It is defined as an explicit type to reduce cross-confusion with the id +// of an event in the queue itself, which is a queue.EntryID. +type producerID uint64 + type produceState struct { cb ackHandler - dropCB func(beat.Event) + dropCB func(interface{}) cancelled bool - lastACK uint32 + lastACK producerID } type ackHandler func(count int) -func newProducer(b *broker, cb ackHandler, dropCB func(beat.Event), dropOnCancel bool) queue.Producer { +func newProducer(b *broker, cb ackHandler, dropCB func(interface{}), dropOnCancel bool) queue.Producer { openState := openState{ log: b.logger, done: make(chan struct{}), @@ -59,7 +64,7 @@ func newProducer(b *broker, cb ackHandler, dropCB func(beat.Event), dropOnCancel } if cb != nil { - p := &ackProducer{broker: b, seq: 1, dropOnCancel: dropOnCancel, openState: openState} + p := &ackProducer{broker: b, dropOnCancel: dropOnCancel, openState: openState} p.state.cb = cb p.state.dropCB = dropCB return p @@ -67,12 +72,19 @@ func newProducer(b *broker, cb ackHandler, dropCB func(beat.Event), dropOnCancel return &forgetfulProducer{broker: b, openState: openState} } -func (p *forgetfulProducer) Publish(event interface{}) bool { - return p.openState.publish(pushRequest{event: event}) +func (p *forgetfulProducer) makePushRequest(event interface{}) pushRequest { + resp := make(chan queue.EntryID, 1) + return pushRequest{ + event: event, + resp: resp} +} + +func (p *forgetfulProducer) Publish(event interface{}) (queue.EntryID, bool) { + return p.openState.publish(p.makePushRequest(event)) } -func (p *forgetfulProducer) TryPublish(event interface{}) bool { - return p.openState.tryPublish(pushRequest{event: event}) +func (p *forgetfulProducer) TryPublish(event interface{}) (queue.EntryID, bool) { + return p.openState.tryPublish(p.makePushRequest(event)) } func (p *forgetfulProducer) Cancel() int { @@ -80,28 +92,31 @@ func (p *forgetfulProducer) Cancel() int { return 0 } -func (p *ackProducer) Publish(event interface{}) bool { - return p.updSeq(p.openState.publish(p.makeRequest(event))) +func (p *ackProducer) makePushRequest(event interface{}) pushRequest { + resp := make(chan queue.EntryID, 1) + return pushRequest{ + event: event, + producer: p, + // We add 1 to the id so the default lastACK of 0 is a + // valid initial state and 1 is the first real id. + producerID: producerID(p.producedCount + 1), + resp: resp} } -func (p *ackProducer) TryPublish(event interface{}) bool { - return p.updSeq(p.openState.tryPublish(p.makeRequest(event))) -} - -func (p *ackProducer) updSeq(ok bool) bool { - if ok { - p.seq++ +func (p *ackProducer) Publish(event interface{}) (queue.EntryID, bool) { + id, published := p.openState.publish(p.makePushRequest(event)) + if published { + p.producedCount++ } - return ok + return id, published } -func (p *ackProducer) makeRequest(event interface{}) pushRequest { - req := pushRequest{ - event: event, - seq: p.seq, - state: &p.state, +func (p *ackProducer) TryPublish(event interface{}) (queue.EntryID, bool) { + id, published := p.openState.tryPublish(p.makePushRequest(event)) + if published { + p.producedCount++ } - return req + return id, published } func (p *ackProducer) Cancel() int { @@ -110,8 +125,8 @@ func (p *ackProducer) Cancel() int { if p.dropOnCancel { ch := make(chan producerCancelResponse) p.broker.cancelChan <- producerCancelRequest{ - state: &p.state, - resp: ch, + producer: p, + resp: ch, } // wait for cancel to being processed @@ -125,25 +140,25 @@ func (st *openState) Close() { close(st.done) } -func (st *openState) publish(req pushRequest) bool { +func (st *openState) publish(req pushRequest) (queue.EntryID, bool) { select { case st.events <- req: - return true + return <-req.resp, true case <-st.done: st.events = nil - return false + return 0, false } } -func (st *openState) tryPublish(req pushRequest) bool { +func (st *openState) tryPublish(req pushRequest) (queue.EntryID, bool) { select { case st.events <- req: - return true + return <-req.resp, true case <-st.done: st.events = nil - return false + return 0, false default: - st.log.Debugf("Dropping event, queue is blocked (seq=%v) ", req.seq) - return false + st.log.Debugf("Dropping event, queue is blocked") + return 0, false } } diff --git a/libbeat/publisher/queue/memqueue/queue_test.go b/libbeat/publisher/queue/memqueue/queue_test.go index 5c810ac18ab..c52f7bdd95b 100644 --- a/libbeat/publisher/queue/memqueue/queue_test.go +++ b/libbeat/publisher/queue/memqueue/queue_test.go @@ -22,6 +22,7 @@ import ( "fmt" "math" "math/rand" + "sync" "testing" "time" @@ -175,3 +176,165 @@ func TestAdjustInputQueueSize(t *testing.T) { assert.Equal(t, int(float64(mainQueue)*maxInputQueueSizeRatio), AdjustInputQueueSize(mainQueue, mainQueue)) }) } + +func TestEntryIDs(t *testing.T) { + entryCount := 100 + + testForward := func(q queue.Queue) { + waiter := &producerACKWaiter{} + producer := q.Producer(queue.ProducerConfig{ACK: waiter.ack}) + for i := 0; i < entryCount; i++ { + id, success := producer.Publish(nil) + assert.Equal(t, success, true, "Queue publish should succeed") + assert.Equal(t, id, queue.EntryID(i), "Entry ID should match publication order") + } + + for i := 0; i < entryCount; i++ { + batch, err := q.Get(1) + assert.NilError(t, err, "Queue read should succeed") + assert.Equal(t, batch.Count(), 1, "Returned batch should have 1 entry") + assert.Equal(t, batch.ID(0), queue.EntryID(i), "Consumed entry IDs should be ordered the same as when they were produced") + + metrics, err := q.Metrics() + assert.NilError(t, err, "Queue metrics call should succeed") + assert.Equal(t, metrics.OldestEntryID, queue.EntryID(i), + fmt.Sprintf("Oldest entry ID before ACKing event %v should be %v", i, i)) + + batch.Done() + waiter.waitForEvents(1) + metrics, err = q.Metrics() + assert.NilError(t, err, "Queue metrics call should succeed") + assert.Equal(t, metrics.OldestEntryID, queue.EntryID(i+1), + fmt.Sprintf("Oldest entry ID after ACKing event %v should be %v", i, i+1)) + + } + } + + testBackward := func(q queue.Queue) { + waiter := &producerACKWaiter{} + producer := q.Producer(queue.ProducerConfig{ACK: waiter.ack}) + for i := 0; i < entryCount; i++ { + id, success := producer.Publish(nil) + assert.Equal(t, success, true, "Queue publish should succeed") + assert.Equal(t, id, queue.EntryID(i), "Entry ID should match publication order") + } + + batches := []queue.Batch{} + + for i := 0; i < entryCount; i++ { + batch, err := q.Get(1) + assert.NilError(t, err, "Queue read should succeed") + assert.Equal(t, batch.Count(), 1, "Returned batch should have 1 entry") + assert.Equal(t, batch.ID(0), queue.EntryID(i), "Consumed entry IDs should be ordered the same as when they were produced") + batches = append(batches, batch) + } + + for i := entryCount - 1; i > 0; i-- { + batches[i].Done() + + // It's hard to remove this delay since the Done signal is propagated + // asynchronously to the queue, and since this test is ensuring that the + // queue _doesn't_ advance we can't use a callback to gate the comparison + // like we do in testForward. However: + // - While this race condition could sometimes let a buggy implementation + // pass, it will not produce a false failure (so it won't contribute + // to general test flakiness) + // - That notwithstanding, when the ACK _does_ cause an incorrect + // metrics update, this delay is enough to recognize it approximately + // 100% of the time, so this test is still a good signal despite + // the slight nondeterminism. + time.Sleep(1 * time.Millisecond) + metrics, err := q.Metrics() + assert.NilError(t, err, "Queue metrics call should succeed") + assert.Equal(t, metrics.OldestEntryID, queue.EntryID(0), + fmt.Sprintf("Oldest entry ID after ACKing event %v should be 0", i)) + } + // ACK the first batch, which should unblock all the later ones + batches[0].Done() + waiter.waitForEvents(100) + metrics, err := q.Metrics() + assert.NilError(t, err, "Queue metrics call should succeed") + assert.Equal(t, metrics.OldestEntryID, queue.EntryID(100), + fmt.Sprintf("Oldest entry ID after ACKing event 0 should be %v", queue.EntryID(entryCount))) + + } + + t.Run("acking in forward order with directEventLoop reports the right event IDs", func(t *testing.T) { + testQueue := NewQueue(nil, Settings{Events: 1000}) + testForward(testQueue) + }) + + t.Run("acking in reverse order with directEventLoop reports the right event IDs", func(t *testing.T) { + testQueue := NewQueue(nil, Settings{Events: 1000}) + testBackward(testQueue) + }) + + t.Run("acking in forward order with bufferedEventLoop reports the right event IDs", func(t *testing.T) { + testQueue := NewQueue(nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}) + testForward(testQueue) + }) + + t.Run("acking in reverse order with bufferedEventLoop reports the right event IDs", func(t *testing.T) { + testQueue := NewQueue(nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}) + testBackward(testQueue) + }) +} + +// producerACKWaiter is a helper that can listen to queue producer callbacks +// and wait on them from the test thread, so we can test the queue's asynchronous +// behavior without relying on time.Sleep. +type producerACKWaiter struct { + sync.Mutex + + // The number of acks received from a producer callback. + acked int + + // The number of acks that callers have waited for in waitForEvents. + waited int + + // When non-nil, this channel is being listened to by a test thread + // blocking on ACKs, and incoming producer callbacks are forwarded + // to it. + ackChan chan int +} + +func (w *producerACKWaiter) ack(count int) { + w.Lock() + defer w.Unlock() + w.acked += count + if w.ackChan != nil { + w.ackChan <- count + } +} + +func (w *producerACKWaiter) waitForEvents(count int) { + w.Lock() + defer w.Unlock() + if w.ackChan != nil { + panic("don't call producerACKWaiter.waitForEvents from multiple goroutines") + } + + avail := w.acked - w.waited + if count <= avail { + w.waited += count + return + } + w.waited = w.acked + count -= avail + // We have advanced as far as we can, we have to wait for + // more incoming ACKs. + // Set a listener and unlock, so ACKs can come in on another + // goroutine. + w.ackChan = make(chan int) + w.Unlock() + + newAcked := 0 + for newAcked < count { + newAcked += <-w.ackChan + } + // When we're done, turn off the listener channel and update + // the number of events waited on. + w.Lock() + w.ackChan = nil + w.waited += count +} diff --git a/libbeat/publisher/queue/memqueue/ringbuf.go b/libbeat/publisher/queue/memqueue/ringbuf.go index aceea52ec1c..d593cc2351b 100644 --- a/libbeat/publisher/queue/memqueue/ringbuf.go +++ b/libbeat/publisher/queue/memqueue/ringbuf.go @@ -52,11 +52,6 @@ type region struct { size int } -type clientState struct { - seq uint32 // event sequence number - state *produceState // the producer it's state used to compute and signal the ACK count -} - func (b *ringBuffer) init(logger *logp.Logger, size int) { *b = ringBuffer{ logger: logger, @@ -64,17 +59,9 @@ func (b *ringBuffer) init(logger *logp.Logger, size int) { } } -// Old spec: -// Returns the number of free entries left in the queue buffer after -// insertion. -// Also returns 0 if there is no space left in the queue to insert -// the given event. However, this is an error state: the first time -// it returns 0, insertion should be disabled by setting the -// pushRequest channel in directEventLoop to nil. -// New spec: // Returns true if the ringBuffer is full after handling // the given insertion, false otherwise. -func (b *ringBuffer) insert(event interface{}, client clientState) { +func (b *ringBuffer) insert(entry queueEntry) { // always insert into region B, if region B exists. // That is, we have 2 regions and region A is currently processed by consumers if b.regB.size > 0 { @@ -83,7 +70,7 @@ func (b *ringBuffer) insert(event interface{}, client clientState) { idx := b.regB.index + b.regB.size avail := b.regA.index - idx if avail > 0 { - b.entries[idx] = queueEntry{event, client} + b.entries[idx] = entry b.regB.size++ } return @@ -97,24 +84,24 @@ func (b *ringBuffer) insert(event interface{}, client clientState) { // If there is space before region A, create // region B there. b.regB = region{index: 0, size: 1} - b.entries[0] = queueEntry{event, client} + b.entries[0] = entry } return } // space available in region A -> let's append the event // log.Debug(" - push into region A") - b.entries[idx] = queueEntry{event, client} + b.entries[idx] = entry b.regA.size++ } // cancel removes all buffered events matching `st`, not yet reserved by // any consumer -func (b *ringBuffer) cancel(st *produceState) int { - cancelledB := b.cancelRegion(st, b.regB) +func (b *ringBuffer) cancel(producer *ackProducer) int { + cancelledB := b.cancelRegion(producer, b.regB) b.regB.size -= cancelledB - cancelledA := b.cancelRegion(st, region{ + cancelledA := b.cancelRegion(producer, region{ index: b.regA.index + b.reserved, size: b.regA.size - b.reserved, }) @@ -126,7 +113,7 @@ func (b *ringBuffer) cancel(st *produceState) int { // cancelRegion removes the events in the specified range having // the specified produceState. It returns the number of events // removed. -func (b *ringBuffer) cancelRegion(st *produceState, reg region) int { +func (b *ringBuffer) cancelRegion(producer *ackProducer, reg region) int { start := reg.index end := start + reg.size entries := b.entries[start:end] @@ -135,7 +122,7 @@ func (b *ringBuffer) cancelRegion(st *produceState, reg region) int { // filter loop for i := 0; i < reg.size; i++ { - if entries[i].client.state == st { + if entries[i].producer == producer { continue // remove } toEntries = append(toEntries, entries[i]) @@ -214,3 +201,10 @@ func (b *ringBuffer) Size() int { func (b *ringBuffer) Items() int { return b.regA.size + b.regB.size } + +func (b *ringBuffer) OldestEntry() *queueEntry { + if b.regA.size == 0 { + return nil + } + return &b.entries[b.regA.index] +} diff --git a/libbeat/publisher/queue/queue.go b/libbeat/publisher/queue/queue.go index 1ca5fda6e14..ce87e6e059c 100644 --- a/libbeat/publisher/queue/queue.go +++ b/libbeat/publisher/queue/queue.go @@ -20,7 +20,6 @@ package queue import ( "errors" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -51,6 +50,10 @@ type Metrics struct { //OldestActiveTimestamp is the timestamp of the oldest item in the queue. OldestActiveTimestamp common.Time + + // OldestActiveID is ID of the oldest unacknowledged event in the queue, or + // the next ID that will be assigned if the queue is empty. + OldestEntryID EntryID } // ErrMetricsNotImplemented is a hopefully temporary type to mark queue metrics as not yet implemented @@ -102,26 +105,28 @@ type ProducerConfig struct { // with close happening early might result in the event being dropped. The callback // gives a queue user a chance to keep track of total number of events // being buffered by the queue. - OnDrop func(beat.Event) + OnDrop func(interface{}) // DropOnCancel is a hint to the queue to drop events if the producer disconnects // via Cancel. DropOnCancel bool } +type EntryID uint64 + // Producer is an interface to be used by the pipelines client to forward // events to a queue. type Producer interface { // Publish adds an event to the queue, blocking if necessary, and returns - // true on success. - Publish(event interface{}) bool + // the new entry's id and true on success. + Publish(event interface{}) (EntryID, bool) // TryPublish adds an event to the queue if doing so will not block the // caller, otherwise it immediately returns. The reasons a publish attempt // might block are defined by the specific queue implementation and its - // configuration. Returns true if the event was successfully added, false - // otherwise. - TryPublish(event interface{}) bool + // configuration. If the event was successfully added, returns true with + // the event's assigned ID, and false otherwise. + TryPublish(event interface{}) (EntryID, bool) // Cancel closes this Producer endpoint. If the producer is configured to // drop its events on Cancel, the number of dropped events is returned. @@ -135,6 +140,7 @@ type Producer interface { // queue that the batch has been consumed and its events can be discarded. type Batch interface { Count() int - Event(i int) interface{} + Entry(i int) interface{} + ID(i int) EntryID Done() } diff --git a/libbeat/publisher/queue/queuetest/producer_cancel.go b/libbeat/publisher/queue/queuetest/producer_cancel.go index 9b5671e1fee..9aacedeb208 100644 --- a/libbeat/publisher/queue/queuetest/producer_cancel.go +++ b/libbeat/publisher/queue/queuetest/producer_cancel.go @@ -82,7 +82,7 @@ func TestProducerCancelRemovesEvents(t *testing.T, factory QueueFactory) { } for i := 0; i < batch.Count(); i++ { - events = append(events, batch.Event(i)) + events = append(events, batch.Entry(i)) } batch.Done() } diff --git a/libbeat/publisher/queue/queuetest/queuetest.go b/libbeat/publisher/queue/queuetest/queuetest.go index f874f3ec288..96b2310d222 100644 --- a/libbeat/publisher/queue/queuetest/queuetest.go +++ b/libbeat/publisher/queue/queuetest/queuetest.go @@ -256,7 +256,7 @@ func makeProducer( total := 0 ackCB = func(N int) { total += N - log.Debugf("producer ACK: N=%v, total=%v\n", N, total) + log.Debugf("producer ACK: N=%v, total=%v/%v\n", N, total, maxEvents) for i := 0; i < N; i++ { ackWG.Done() diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 63db5965f3b..e1d6c12311a 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -370,7 +370,7 @@ func TestESLoader_Load(t *testing.T) { require.NoError(t, err) p, ok := val.(map[string]interface{}) require.True(t, ok) - var properties []string + properties := make([]string, 0, len(p)) for k := range p { properties = append(properties, k) } diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 0c3e184e0e6..203ea067c52 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -3,7 +3,7 @@ """ import subprocess - +import random import unittest import os import shutil @@ -396,7 +396,7 @@ def setUp(self): # create working dir self.working_dir = os.path.abspath(os.path.join( - self.build_path + "run", self.id())) + self.build_path, "run", self.id() + str(random.randrange(1000)))) if os.path.exists(self.working_dir): shutil.rmtree(self.working_dir) os.makedirs(self.working_dir) @@ -411,13 +411,25 @@ def setUp(self): # update the last_run link if os.path.islink(self.build_path + "last_run"): os.unlink(self.build_path + "last_run") - os.symlink(self.build_path + f"run/{self.id()}", + os.symlink(self.working_dir, self.build_path + "last_run") except BaseException: # symlink is best effort and can fail when # running tests in parallel pass + # Keep last 5 runs + candidates = [] + to_keep = 5 + for dir_entry in os.listdir(os.path.join(self.build_path, "run")): + if re.search(self.id() + r"[0-9]+$", dir_entry): + candidates.append(dir_entry) + if len(candidates) > to_keep: + candidates.sort(reverse=True, key=lambda dirname: os.path.getmtime( + os.path.join(self.build_path, "run", dirname))) + for d in candidates[to_keep:]: + shutil.rmtree(os.path.join(self.build_path, "run", d)) + def wait_until(self, cond, max_timeout=20, poll_interval=0.1, name="cond", err_msg=""): """ TODO: this can probably be a "wait_until_output_count", among other things, since that could actually use `self`, and this can become an internal function diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 0dc130bba95..27a060d6499 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -22,6 +22,9 @@ jsonschema==3.2.0 kafka-python==1.4.3 MarkupSafe==1.1.1 more-itertools==8.4.0 +opentelemetry-api==1.11.0 +opentelemetry-exporter-otlp==1.11.0 +opentelemetry-sdk==1.11.0 ordered-set==3.1.1 packaging==20.4 parameterized==0.7.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index 1feeb974f40..97a1224445c 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -21,6 +21,9 @@ jsonschema==3.2.0 kafka-python==1.4.3 MarkupSafe==1.1.1 more-itertools==8.4.0 +opentelemetry-api==1.11.0 +opentelemetry-exporter-otlp==1.11.0 +opentelemetry-sdk==1.11.0 ordered-set==3.1.1 packaging==20.4 parameterized==0.7.0 diff --git a/libbeat/version/version.go b/libbeat/version/version.go index 8efa3c36012..b6dbd3b7bda 100644 --- a/libbeat/version/version.go +++ b/libbeat/version/version.go @@ -18,4 +18,4 @@ // Code generated by dev-tools/set_version package version -const defaultBeatVersion = "8.4.0" +const defaultBeatVersion = "8.5.0" diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index cfe746a0be0..4eee6b0249b 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.12 +FROM golang:1.18.5 RUN \ apt update \ diff --git a/metricbeat/Dockerfile.debug b/metricbeat/Dockerfile.debug deleted file mode 100644 index b8876e0af79..00000000000 --- a/metricbeat/Dockerfile.debug +++ /dev/null @@ -1,26 +0,0 @@ -FROM golang:alpine3.15 as builder - -ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin - -ENV CGO_ENABLED=0 - -RUN apk add --no-cache git -RUN go install github.com/go-delve/delve/cmd/dlv@v1.8.3 - -COPY metricbeat /usr/share/metricbeat/metricbeat - -FROM alpine:3.15 - -ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin - -WORKDIR /usr/share/metricbeat - -ENV ELASTICSEARCH_PASSWORD=changeme -ENV ELASTICSEARCH_USERNAME=elastic -ENV ELASTICSEARCH_HOST=elasticsearch - -COPY --from=builder /go/bin/dlv /go/bin/dlv -COPY --from=builder /usr/share/metricbeat/metricbeat /usr/share/metricbeat/metricbeat - -ENTRYPOINT ["dlv", "--headless=true", "--listen=:56268", "--api-version=2", "--log", "exec", "./metricbeat", "--"] -CMD [ "-e" ] diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 8832fe259fc..29736ea8c42 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C metricbeat check; - make -C metricbeat update; - make -C x-pack/metricbeat check; - make -C x-pack/metricbeat update; - make check-no-changes; - stage: checks unitTest: mage: "mage build unitTest" stage: mandatory diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index df5c7655967..8b204a3cad3 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -18,10 +18,9 @@ package beater import ( + "fmt" "sync" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" @@ -131,7 +130,7 @@ func DefaultTestModulesCreator() beat.Creator { func newMetricbeat(b *beat.Beat, c *conf.C, options ...Option) (*Metricbeat, error) { config := defaultConfig if err := c.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "error reading configuration file") + return nil, fmt.Errorf("error reading configuration file: %w", err) } dynamicCfgEnabled := config.ConfigModules.Enabled() || config.Autodiscover != nil || b.Manager.Enabled() diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b924edddc52..e972654ff07 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4799,6 +4799,24 @@ type: long -- +*`aws.transitgateway.metrics.BytesDropCountNoRoute.sum`*:: ++ +-- +The number of bytes dropped because they did not match a route. + +type: long + +-- + +*`aws.transitgateway.metrics.BytesDropCountBlackhole.sum`*:: ++ +-- +The number of bytes dropped because they matched a blackhole route. + +type: long + +-- + [float] === usage @@ -35553,6 +35571,42 @@ type: long -- +*`gcp.dataproc.cluster.job.completion_time.value`*:: ++ +-- +The time jobs took to complete from the time the user submits a job to the time Dataproc reports it is completed. + +type: object + +-- + +*`gcp.dataproc.cluster.job.duration.value`*:: ++ +-- +The time jobs have spent in a given state. + +type: object + +-- + +*`gcp.dataproc.cluster.operation.completion_time.value`*:: ++ +-- +The time operations took to complete from the time the user submits a operation to the time Dataproc reports it is completed. + +type: object + +-- + +*`gcp.dataproc.cluster.operation.duration.value`*:: ++ +-- +The time operations have spent in a given state. + +type: object + +-- + [float] === firestore @@ -36129,6 +36183,96 @@ type: long -- +*`gcp.loadbalancing.https.backend_latencies.value`*:: ++ +-- +A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + +type: object + +-- + +*`gcp.loadbalancing.https.external.regional.backend_latencies.value`*:: ++ +-- +A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + +type: object + +-- + +*`gcp.loadbalancing.https.external.regional.total_latencies.value`*:: ++ +-- +A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. + +type: object + +-- + +*`gcp.loadbalancing.https.frontend_tcp_rtt.value`*:: ++ +-- +A distribution of the RTT measured for each connection between client and proxy. + +type: object + +-- + +*`gcp.loadbalancing.https.internal.backend_latencies.value`*:: ++ +-- +A distribution of the latency calculated from when the request was sent by the internal HTTP/S load balancer proxy to the backend until the proxy received from the backend the last byte of response. + +type: object + +-- + +*`gcp.loadbalancing.https.internal.total_latencies.value`*:: ++ +-- +A distribution of the latency calculated from when the request was received by the internal HTTP/S load balancer proxy until the proxy got ACK from client on last response byte. + +type: object + +-- + +*`gcp.loadbalancing.https.total_latencies.value`*:: ++ +-- +A distribution of the latency calculated from when the request was received by the external HTTP/S load balancer proxy until the proxy got ACK from client on last response byte. + +type: object + +-- + +*`gcp.loadbalancing.l3.external.rtt_latencies.value`*:: ++ +-- +A distribution of the round trip time latency, measured over TCP connections for the external network load balancer. + +type: object + +-- + +*`gcp.loadbalancing.l3.internal.rtt_latencies.value`*:: ++ +-- +A distribution of RTT measured over TCP connections for internal TCP/UDP load balancer flows. + +type: object + +-- + +*`gcp.loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt.value`*:: ++ +-- +A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client. + +type: object + +-- + [float] === pubsub @@ -36369,15 +36513,6 @@ type: long -- -*`gcp.pubsub.subscription.push_request_latencies.value`*:: -+ --- -Distribution of push request latencies (in microseconds), grouped by result. - -type: long - --- - *`gcp.pubsub.subscription.retained_acked.bytes`*:: + -- @@ -36500,7 +36635,7 @@ type: long -- Distribution of publish message sizes (in bytes) -type: long +type: object -- @@ -36567,6 +36702,24 @@ type: long -- +*`gcp.pubsub.subscription.ack_latencies.value`*:: ++ +-- +Distribution of ack latencies in milliseconds. The ack latency is the time between when Cloud Pub/Sub sends a message to a subscriber client and when Cloud Pub/Sub receives an Acknowledge request for that message. + +type: object + +-- + +*`gcp.pubsub.subscription.push_request_latencies.value`*:: ++ +-- +Distribution of push request latencies (in microseconds), grouped by result. + +type: object + +-- + [float] === storage @@ -43109,7 +43262,7 @@ format: percent *`kubernetes.container.cpu.usage.limit.pct`*:: + -- -CPU usage as a percentage of the defined limit for the container (or total node allocatable CPU if unlimited) +CPU usage as a percentage of the defined limit for the container (or total node allocatable CPU if unlimited). If the container CPU limits are missing and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. type: scaled_float @@ -43237,7 +43390,7 @@ format: percent *`kubernetes.container.memory.usage.limit.pct`*:: + -- -Memory usage as a percentage of the defined limit for the container (or total node allocatable memory if unlimited) +Memory usage as a percentage of the defined limit for the container (or total node allocatable memory if unlimited). If the container Memory limits are missing and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. type: scaled_float @@ -43502,7 +43655,7 @@ type: long -- -*`kubernetes.controllermanager.client.request.duration.sec.bucket.*`*:: +*`kubernetes.controllermanager.client.request.duration.us.bucket.*`*:: + -- Response latency distribution, histogram buckets @@ -43511,16 +43664,16 @@ type: object -- -*`kubernetes.controllermanager.client.request.duration.sec.sum`*:: +*`kubernetes.controllermanager.client.request.duration.us.sum`*:: + -- -Request duration, sum in seconds +Request duration, sum in microseconds type: long -- -*`kubernetes.controllermanager.client.request.duration.sec.count`*:: +*`kubernetes.controllermanager.client.request.duration.us.count`*:: + -- Request duration, number of operations @@ -43575,33 +43728,6 @@ type: long -- -*`kubernetes.controllermanager.workqueue.work.duration.sec.sum`*:: -+ --- -Processing time of workqueue item, sum in seconds - -type: long - --- - -*`kubernetes.controllermanager.workqueue.work.duration.sec.count`*:: -+ --- -Processing time of workqueue item, number of operations - -type: long - --- - -*`kubernetes.controllermanager.workqueue.work.duration.sec.bucket.*`*:: -+ --- -Processing time of workqueue item, histogram buckets - -type: object - --- - *`kubernetes.controllermanager.node.collector.eviction.count`*:: + @@ -44254,7 +44380,7 @@ format: percent *`kubernetes.pod.cpu.usage.limit.pct`*:: + -- -CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited) +CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited). If one or more containers of the pod is unlimited and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. type: scaled_float @@ -44292,7 +44418,7 @@ format: percent *`kubernetes.pod.memory.usage.limit.pct`*:: + -- -Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited) +Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited). If one or more containers of the pod is unlimited and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. type: scaled_float diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 8a215dcc6fd..372599480bc 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -230,7 +230,12 @@ func PythonIntegTest(ctx context.Context) error { if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } - runner, err := devtools.NewDockerIntegrationRunner(devtools.ListMatchingEnvVars("PYTEST_")...) + + passThroughEnvVars := append( + []string{"ELASTICSEARCH_VERSION", "KIBANA_VERSION", "BEAT_VERSION"}, + devtools.ListMatchingEnvVars("PYTEST_")..., + ) + runner, err := devtools.NewDockerIntegrationRunner(passThroughEnvVars...) if err != nil { return err } diff --git a/metricbeat/module/ceph/test_ceph.py b/metricbeat/module/ceph/test_ceph.py index 91bb6e56e7c..b95e0314351 100644 --- a/metricbeat/module/ceph/test_ceph.py +++ b/metricbeat/module/ceph/test_ceph.py @@ -8,6 +8,7 @@ @metricbeat.parameterized_with_supported_versions +@unittest.skip("broken test suite: https://github.com/elastic/beats/issues/32898") class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['ceph'] diff --git a/metricbeat/module/jolokia/_meta/Dockerfile b/metricbeat/module/jolokia/_meta/Dockerfile index d79cf4640cc..769749489ac 100644 --- a/metricbeat/module/jolokia/_meta/Dockerfile +++ b/metricbeat/module/jolokia/_meta/Dockerfile @@ -1,5 +1,5 @@ # Tomcat is started to fetch Jolokia metrics from it -FROM java:8-jdk-alpine +FROM openjdk:8-jdk-alpine ENV TOMCAT_VERSION 7.0.86 ENV TC apache-tomcat-${TOMCAT_VERSION} diff --git a/metricbeat/module/kubernetes/_meta/kibana/7/dashboard/2ec26ce0-f5f1-11ec-8853-8b596bddf5f9.json b/metricbeat/module/kubernetes/_meta/kibana/7/dashboard/2ec26ce0-f5f1-11ec-8853-8b596bddf5f9.json index d5154163ba5..f0f1bcbacb0 100644 --- a/metricbeat/module/kubernetes/_meta/kibana/7/dashboard/2ec26ce0-f5f1-11ec-8853-8b596bddf5f9.json +++ b/metricbeat/module/kubernetes/_meta/kibana/7/dashboard/2ec26ce0-f5f1-11ec-8853-8b596bddf5f9.json @@ -90,10 +90,10 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Request Latency in Seconds", + "label": "Request Latency in Milliseconds", "operationType": "formula", "params": { - "formula": "(sum(kubernetes.controllermanager.client.request.duration.sec.sum)/ sum(kubernetes.controllermanager.client.request.duration.sec.count))", + "formula": "(sum(kubernetes.controllermanager.client.request.duration.us.sum)/ sum(kubernetes.controllermanager.client.request.duration.us.count))/1000", "isFormulaBroken": false }, "references": [ @@ -105,44 +105,56 @@ "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Client Request Latency in Seconds", + "label": "Part of Request Latency in Milliseconds", "operationType": "sum", "params": { "emptyAsNull": false }, "scale": "ratio", - "sourceField": "kubernetes.controllermanager.client.request.duration.sec.sum" + "sourceField": "kubernetes.controllermanager.client.request.duration.us.sum" }, "43097f7a-e478-47bc-81c1-7541bd899d46X1": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Client Request Latency in Seconds", + "label": "Part of Request Latency in Milliseconds", "operationType": "sum", "params": { "emptyAsNull": false }, "scale": "ratio", - "sourceField": "kubernetes.controllermanager.client.request.duration.sec.count" + "sourceField": "kubernetes.controllermanager.client.request.duration.us.count" }, "43097f7a-e478-47bc-81c1-7541bd899d46X2": { "customLabel": true, "dataType": "number", "isBucketed": false, - "label": "Part of Client Request Latency in Seconds", + "label": "Part of Request Latency in Milliseconds", "operationType": "math", "params": { "tinymathAst": { "args": [ - "43097f7a-e478-47bc-81c1-7541bd899d46X0", - "43097f7a-e478-47bc-81c1-7541bd899d46X1" + { + "args": [ + "43097f7a-e478-47bc-81c1-7541bd899d46X0", + "43097f7a-e478-47bc-81c1-7541bd899d46X1" + ], + "location": { + "max": 133, + "min": 1 + }, + "name": "divide", + "text": "sum(kubernetes.controllermanager.client.request.duration.us.sum)/ sum(kubernetes.controllermanager.client.request.duration.us.count)", + "type": "function" + }, + 1000 ], "location": { - "max": 136, + "max": 139, "min": 0 }, "name": "divide", - "text": "(sum(kubernetes.controllermanager.client.request.duration.sec.sum)/ sum(kubernetes.controllermanager.client.request.duration.sec.count))", + "text": "(sum(kubernetes.controllermanager.client.request.duration.us.sum)/ sum(kubernetes.controllermanager.client.request.duration.us.count))/1000", "type": "function" } }, @@ -174,7 +186,7 @@ "filters": [], "query": { "language": "kuery", - "query": "NOT (kubernetes.apiserver.request.verb: WATCH or kubernetes.apiserver.request.verb: CONNECT)" + "query": "" }, "visualization": { "layers": [ @@ -219,7 +231,7 @@ "panelIndex": "1bd24fa1-319e-4cae-9d45-d821b06a8034", "title": "Average Request Latency per Url [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -333,7 +345,7 @@ "panelIndex": "1604f0de-edd6-456e-8670-ab9b33988abb", "title": "Controller CPU [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -465,7 +477,7 @@ "panelIndex": "303702e1-ba33-49f2-b337-4cc7d7305606", "title": "Controller Memory in Bytes [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -606,7 +618,7 @@ "panelIndex": "74dcc137-b625-44d5-ae91-072040ef4b0a", "title": "Controller Workqueue adds [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -747,7 +759,7 @@ "panelIndex": "d680bbc4-b3dd-4237-9825-5394ff9d402c", "title": "Controller Workqueue retries [Metricbeat Kubernetes] ", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -874,12 +886,12 @@ "i": "0be041d4-096a-4131-ab5b-f9ae02eb685a", "w": 24, "x": 0, - "y": 59 + "y": 44 }, "panelIndex": "0be041d4-096a-4131-ab5b-f9ae02eb685a", "title": "Controller Longest running processor [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -1011,194 +1023,7 @@ "panelIndex": "a015a089-006e-4eb8-ad45-7f9d5f16eecb", "title": "Controller Unfinished jobs sec [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" - }, - { - "embeddableConfig": { - "attributes": { - "references": [ - { - "id": "metricbeat-*", - "name": "indexpattern-datasource-layer-f7b7d15b-f8d9-4c06-abf0-7503ae32b8e9", - "type": "index-pattern" - } - ], - "state": { - "datasourceStates": { - "indexpattern": { - "layers": { - "f7b7d15b-f8d9-4c06-abf0-7503ae32b8e9": { - "columnOrder": [ - "35a11916-4ca3-421b-9df2-521f52f21fbb", - "ed3c7efa-0467-4a57-8d06-0f4775906cc5", - "43097f7a-e478-47bc-81c1-7541bd899d46", - "43097f7a-e478-47bc-81c1-7541bd899d46X0", - "43097f7a-e478-47bc-81c1-7541bd899d46X1", - "43097f7a-e478-47bc-81c1-7541bd899d46X2" - ], - "columns": { - "35a11916-4ca3-421b-9df2-521f52f21fbb": { - "dataType": "string", - "isBucketed": true, - "label": "Top 50 values of kubernetes.controllermanager.name", - "operationType": "terms", - "params": { - "missingBucket": false, - "orderBy": { - "fallback": true, - "type": "alphabetical" - }, - "orderDirection": "asc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "secondaryFields": [], - "size": 50 - }, - "scale": "ordinal", - "sourceField": "kubernetes.controllermanager.name" - }, - "43097f7a-e478-47bc-81c1-7541bd899d46": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Workqueue Duration in Seconds", - "operationType": "formula", - "params": { - "formula": "(sum(kubernetes.controllermanager.workqueue.work.duration.sec.sum)/ sum(kubernetes.controllermanager.workqueue.work.duration.sec.count))", - "isFormulaBroken": false - }, - "references": [ - "43097f7a-e478-47bc-81c1-7541bd899d46X2" - ], - "scale": "ratio" - }, - "43097f7a-e478-47bc-81c1-7541bd899d46X0": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Request Latency in Seconds", - "operationType": "sum", - "params": { - "emptyAsNull": false - }, - "scale": "ratio", - "sourceField": "kubernetes.controllermanager.workqueue.work.duration.sec.sum" - }, - "43097f7a-e478-47bc-81c1-7541bd899d46X1": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Request Latency in Seconds", - "operationType": "sum", - "params": { - "emptyAsNull": false - }, - "scale": "ratio", - "sourceField": "kubernetes.controllermanager.workqueue.work.duration.sec.count" - }, - "43097f7a-e478-47bc-81c1-7541bd899d46X2": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Part of Request Latency in Seconds", - "operationType": "math", - "params": { - "tinymathAst": { - "args": [ - "43097f7a-e478-47bc-81c1-7541bd899d46X0", - "43097f7a-e478-47bc-81c1-7541bd899d46X1" - ], - "location": { - "max": 136, - "min": 0 - }, - "name": "divide", - "text": "(sum(kubernetes.controllermanager.workqueue.work.duration.sec.sum)/ sum(kubernetes.controllermanager.workqueue.work.duration.sec.count))", - "type": "function" - } - }, - "references": [ - "43097f7a-e478-47bc-81c1-7541bd899d46X0", - "43097f7a-e478-47bc-81c1-7541bd899d46X1" - ], - "scale": "ratio" - }, - "ed3c7efa-0467-4a57-8d06-0f4775906cc5": { - "dataType": "date", - "isBucketed": true, - "label": "@timestamp", - "operationType": "date_histogram", - "params": { - "dropPartials": false, - "includeEmptyRows": true, - "interval": "auto" - }, - "scale": "interval", - "sourceField": "@timestamp" - } - }, - "incompleteColumns": {} - } - } - } - }, - "filters": [], - "query": { - "language": "kuery", - "query": "NOT (kubernetes.apiserver.request.verb: WATCH or kubernetes.apiserver.request.verb: CONNECT)" - }, - "visualization": { - "axisTitlesVisibilitySettings": { - "x": true, - "yLeft": false, - "yRight": true - }, - "layers": [ - { - "accessors": [ - "43097f7a-e478-47bc-81c1-7541bd899d46" - ], - "layerId": "f7b7d15b-f8d9-4c06-abf0-7503ae32b8e9", - "layerType": "data", - "position": "top", - "seriesType": "area", - "showGridlines": false, - "splitAccessor": "35a11916-4ca3-421b-9df2-521f52f21fbb", - "xAccessor": "ed3c7efa-0467-4a57-8d06-0f4775906cc5" - } - ], - "legend": { - "isVisible": true, - "legendSize": "large", - "position": "right" - }, - "preferredSeriesType": "area", - "title": "Empty XY chart", - "valueLabels": "hide", - "valuesInLegend": true, - "yTitle": "" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" - }, - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 15, - "i": "3e8953f2-7543-4a5d-ac80-f495869c8b2b", - "w": 24, - "x": 0, - "y": 44 - }, - "panelIndex": "3e8953f2-7543-4a5d-ac80-f495869c8b2b", - "title": "Average Duration of Item in Workqueue in sec [Metricbeat Kubernetes]", - "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -1375,13 +1200,13 @@ "h": 15, "i": "bf28ccd1-0c7a-4672-9c32-0576e8b0c67f", "w": 24, - "x": 24, + "x": 0, "y": 59 }, "panelIndex": "bf28ccd1-0c7a-4672-9c32-0576e8b0c67f", "title": "NodeController Informations [Metricbeat Kubernetes]", "type": "lens", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" }, { "embeddableConfig": { @@ -1389,15 +1214,15 @@ }, "gridData": { "h": 14, - "i": "77d1de0c-5afd-436c-a875-f4d2ed01dccb", + "i": "b5aa99d0-ac4d-4706-af16-057fa5260467", "w": 16, "x": 0, "y": 0 }, - "panelIndex": "77d1de0c-5afd-436c-a875-f4d2ed01dccb", - "panelRefName": "panel_77d1de0c-5afd-436c-a875-f4d2ed01dccb", + "panelIndex": "b5aa99d0-ac4d-4706-af16-057fa5260467", + "panelRefName": "panel_b5aa99d0-ac4d-4706-af16-057fa5260467", "type": "visualization", - "version": "8.4.0-SNAPSHOT" + "version": "8.4.0" } ], "timeRestore": false, @@ -1450,23 +1275,18 @@ "name": "a015a089-006e-4eb8-ad45-7f9d5f16eecb:indexpattern-datasource-layer-dc84d4db-4cef-465b-9db9-5074b7e03b66", "type": "index-pattern" }, - { - "id": "metricbeat-*", - "name": "3e8953f2-7543-4a5d-ac80-f495869c8b2b:indexpattern-datasource-layer-f7b7d15b-f8d9-4c06-abf0-7503ae32b8e9", - "type": "index-pattern" - }, { "id": "metricbeat-*", "name": "bf28ccd1-0c7a-4672-9c32-0576e8b0c67f:indexpattern-datasource-layer-c46cb227-5f12-45ee-a3f0-58f837655aeb", "type": "index-pattern" }, { - "id": "e6f265b0-f5f9-11ec-8853-8b596bddf5f9", - "name": "77d1de0c-5afd-436c-a875-f4d2ed01dccb:panel_77d1de0c-5afd-436c-a875-f4d2ed01dccb", + "id": "2d866300-0cb1-11ed-b5e5-a774dc3eeaba", + "name": "b5aa99d0-ac4d-4706-af16-057fa5260467:panel_b5aa99d0-ac4d-4706-af16-057fa5260467", "type": "visualization" } ], "type": "dashboard", - "updated_at": "2022-06-27T09:17:35.737Z", - "version": "WzU1MDQsM10=" + "updated_at": "2022-07-26T07:04:32.144Z", + "version": "WzEzODcsMV0=" } \ No newline at end of file diff --git a/metricbeat/module/kubernetes/_meta/kibana/7/visualization/2d866300-0cb1-11ed-b5e5-a774dc3eeaba.json b/metricbeat/module/kubernetes/_meta/kibana/7/visualization/2d866300-0cb1-11ed-b5e5-a774dc3eeaba.json new file mode 100644 index 00000000000..62d5a4e33a4 --- /dev/null +++ b/metricbeat/module/kubernetes/_meta/kibana/7/visualization/2d866300-0cb1-11ed-b5e5-a774dc3eeaba.json @@ -0,0 +1,190 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Controller Process [Metricbeat Kubernetes]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 1, + "hide_last_value_indicator": true, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metricbeat-*", + "interval": "30s", + "isModelInvalid": false, + "markdown": "| | |\n|---|---|\n|**Days running**|{{ math.started.last.raw }}|\n|**File descriptors open**|{{ average_of_kubernetes_controllermanager_process_fds_open_count.fds.last.formatted }}|\n|**Maximun File descriptors**|{{ max_of_kubernetes_controllermanager_process_fds_max_count.fds_max.last.formatted }}|\n|**Resident Memory**|{{ average_of_kubernetes_controllermanager_process_memory_resident_bytes.resident_memory.last.formatted }}|\n|**Virtual Memory**|{{ average_of_kubernetes_controllermanager_process_memory_virtual_bytes.virtual_memory.last.formatted }}|\n", + "markdown_css": "table, tr, td, th{border:none}", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "line_width": 1, + "metrics": [ + { + "field": "kubernetes.controllermanager.process.started.sec", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + }, + { + "id": "ebfdb1c0-9c1c-11e9-b29f-d55be9348723", + "script": "round( (params._timestamp /1000 - params.started) / 86400, 2)", + "type": "math", + "variables": [ + { + "field": "61ca57f2-469d-11e7-af02-69e470af7417", + "id": "ef30a230-9c1c-11e9-b29f-d55be9348723", + "name": "started" + } + ] + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range", + "var_name": "started" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "1212ba80-9c1e-11e9-b29f-d55be9348723", + "label": "", + "line_width": 1, + "metrics": [ + { + "field": "kubernetes.controllermanager.process.fds.open.count", + "id": "1212ba81-9c1e-11e9-b29f-d55be9348723", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range", + "var_name": "fds" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "bytes", + "id": "3f3b17a0-9c1e-11e9-b29f-d55be9348723", + "label": "", + "line_width": 1, + "metrics": [ + { + "field": "kubernetes.controllermanager.process.memory.resident.bytes", + "id": "3f3b17a1-9c1e-11e9-b29f-d55be9348723", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range", + "var_name": "resident_memory" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "bytes", + "id": "b1a01b50-9c1f-11e9-b29f-d55be9348723", + "line_width": 1, + "metrics": [ + { + "field": "kubernetes.controllermanager.process.memory.virtual.bytes", + "id": "b1a01b51-9c1f-11e9-b29f-d55be9348723", + "type": "avg" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range", + "var_name": "virtual_memory" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "6faac700-f2dd-11ec-ab74-4342b85d7779", + "label": "", + "line_width": 1, + "metrics": [ + { + "field": "kubernetes.controllermanager.process.fds.max.count", + "id": "6faaee10-f2dd-11ec-ab74-4342b85d7779", + "type": "max" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range", + "var_name": "fds_max" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "time_range_mode": "last_value", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "markdown", + "use_kibana_indexes": false + }, + "title": "Controller Process [Metricbeat Kubernetes]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.4.0", + "id": "2d866300-0cb1-11ed-b5e5-a774dc3eeaba", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-07-26T07:04:23.093Z", + "version": "WzEzNzksMV0=" +} \ No newline at end of file diff --git a/metricbeat/module/kubernetes/_meta/test/stats_summary_multiple_containers.json b/metricbeat/module/kubernetes/_meta/test/stats_summary_multiple_containers.json new file mode 100644 index 00000000000..56490e8c508 --- /dev/null +++ b/metricbeat/module/kubernetes/_meta/test/stats_summary_multiple_containers.json @@ -0,0 +1,163 @@ +{ + "node": { + "nodeName": "gke-beats-default-pool-a5b33e2e-hdww", + "systemContainers": [ + { + "name": "kubelet", + "startTime": "2017-04-18T12:53:49Z", + "cpu": { + "time": "2017-04-20T08:06:46Z", + "usageNanoCores": 11263994, + "usageCoreNanoSeconds": 2357800908948 + }, + "memory": { + "time": "2017-04-20T08:06:46Z", + "usageBytes": 36683776, + "workingSetBytes": 36495360, + "rssBytes": 35512320, + "pageFaults": 100835242, + "majorPageFaults": 0 + }, + "userDefinedMetrics": null + } + ], + "startTime": "2017-04-18T12:53:49Z", + "cpu": { + "time": "2017-04-20T08:06:41Z", + "usageNanoCores": 18691146, + "usageCoreNanoSeconds": 4189523881380 + }, + "memory": { + "time": "2017-04-20T08:06:41Z", + "availableBytes": 1768316928, + "usageBytes": 2764943360, + "workingSetBytes": 2111090688, + "rssBytes": 2150400, + "pageFaults": 131567, + "majorPageFaults": 103 + }, + "network": { + "time": "2017-04-20T08:06:41Z", + "rxBytes": 1115133198, + "rxErrors": 0, + "txBytes": 812729002, + "txErrors": 0 + }, + "fs": { + "availableBytes": 98727014400, + "capacityBytes": 101258067968, + "usedBytes": 2514276352, + "inodesFree": 18446744073709551615, + "inodes": 6258720, + "inodesUsed": 138624 + }, + "runtime": { + "imageFs": { + "availableBytes": 98727014400, + "capacityBytes": 101258067968, + "usedBytes": 860204379, + "inodesFree": 18446744073709551615, + "inodes": 6258720, + "inodesUsed": 138624 + } + } + }, + "pods": [ + { + "podRef": { + "name": "nginx-deployment-2303442956-pcqfc", + "namespace": "default", + "uid": "beabc196-2456-11e7-a3ad-42010a840235" + }, + "startTime": "2017-04-18T16:47:44Z", + "containers": [ + { + "name": "nginx", + "startTime": "2017-04-18T16:47:44Z", + "cpu": { + "time": "2017-04-20T08:06:34Z", + "usageNanoCores": 11263994, + "usageCoreNanoSeconds": 43959424 + }, + "memory": { + "time": "2017-04-20T08:06:34Z", + "usageBytes": 1462272, + "workingSetBytes": 1454080, + "rssBytes": 1409024, + "pageFaults": 841, + "majorPageFaults": 0 + }, + "rootfs": { + "availableBytes": 98727014400, + "capacityBytes": 101258067968, + "usedBytes": 61440, + "inodesFree": 6120096, + "inodes": 6258720, + "inodesUsed": 21 + }, + "logs": { + "availableBytes": 98727014400, + "capacityBytes": 101258067968, + "usedBytes": 28672, + "inodesFree": 6120096, + "inodes": 6258720, + "inodesUsed": 138624 + }, + "userDefinedMetrics": null + }, + { + "name": "sidecar", + "startTime": "2017-04-18T16:47:44Z", + "cpu": { + "time": "2017-04-20T08:06:34Z", + "usageNanoCores": 11263994, + "usageCoreNanoSeconds": 43959424 + }, + "memory": { + "time": "2017-04-20T08:06:34Z", + "usageBytes": 1462272, + "workingSetBytes": 1454080, + "rssBytes": 1409024, + "pageFaults": 841, + "majorPageFaults": 0 + }, + "rootfs": { + "availableBytes": 98727014400, + "capacityBytes": 101258067968, + "usedBytes": 61440, + "inodesFree": 6120096, + "inodes": 6258720, + "inodesUsed": 21 + }, + "logs": { + "availableBytes": 98727014400, + "capacityBytes": 101258067968, + "usedBytes": 28672, + "inodesFree": 6120096, + "inodes": 6258720, + "inodesUsed": 138624 + }, + "userDefinedMetrics": null + } + ], + "network": { + "time": "2017-04-20T08:06:41Z", + "rxBytes": 107056, + "rxErrors": 0, + "txBytes": 72447, + "txErrors": 0 + }, + "volume": [ + { + "availableBytes": 1939689472, + "capacityBytes": 1939701760, + "usedBytes": 12288, + "inodesFree": 473551, + "inodes": 473560, + "inodesUsed": 9, + "name": "default-token-sg8x5" + } + ] + } + ] + } diff --git a/metricbeat/module/kubernetes/container/_meta/fields.yml b/metricbeat/module/kubernetes/container/_meta/fields.yml index 81b99f75c0c..2d9754f628d 100644 --- a/metricbeat/module/kubernetes/container/_meta/fields.yml +++ b/metricbeat/module/kubernetes/container/_meta/fields.yml @@ -36,7 +36,7 @@ type: scaled_float format: percent description: > - CPU usage as a percentage of the defined limit for the container (or total node allocatable CPU if unlimited) + CPU usage as a percentage of the defined limit for the container (or total node allocatable CPU if unlimited). If the container CPU limits are missing and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. - name: logs type: group description: > @@ -109,7 +109,7 @@ type: scaled_float format: percent description: > - Memory usage as a percentage of the defined limit for the container (or total node allocatable memory if unlimited) + Memory usage as a percentage of the defined limit for the container (or total node allocatable memory if unlimited). If the container Memory limits are missing and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. - name: rss type: group fields: diff --git a/metricbeat/module/kubernetes/container/container.go b/metricbeat/module/kubernetes/container/container.go index 5e09d9a7f53..c277406faee 100644 --- a/metricbeat/module/kubernetes/container/container.go +++ b/metricbeat/module/kubernetes/container/container.go @@ -75,7 +75,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, http: http, - enricher: util.NewContainerMetadataEnricher(base, mod.GetPerfMetricsCache(), true), + enricher: util.NewContainerMetadataEnricher(base, mod.GetMetricsRepo(), true), mod: mod, }, nil } @@ -93,7 +93,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { return } - events, err := eventMapping(body, m.mod.GetPerfMetricsCache(), m.Logger()) + events, err := eventMapping(body, m.mod.GetMetricsRepo(), m.Logger()) if err != nil { m.Logger().Error(err) reporter.Error(err) diff --git a/metricbeat/module/kubernetes/container/container_test.go b/metricbeat/module/kubernetes/container/container_test.go index d50ca134b56..16105109613 100644 --- a/metricbeat/module/kubernetes/container/container_test.go +++ b/metricbeat/module/kubernetes/container/container_test.go @@ -24,47 +24,74 @@ import ( "io/ioutil" "os" "testing" - "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) +// this file is used for the tests to compare expected result const testFile = "../_meta/test/stats_summary.json" -func TestEventMapping(t *testing.T) { - logger := logp.NewLogger("kubernetes.container") +type ContainerTestSuite struct { + suite.Suite + MetricsRepo *util.MetricsRepo + NodeName string + Namespace string + PodName string + ContainerName string + PodId util.PodId + Logger *logp.Logger + NodeMetrics *util.NodeMetrics + ContainerMetrics *util.ContainerMetrics +} + +func (s *ContainerTestSuite) SetupTest() { + s.MetricsRepo = util.NewMetricsRepo() + s.NodeName = "gke-beats-default-pool-a5b33e2e-hdww" + s.Namespace = "default" + s.PodName = "nginx-deployment-2303442956-pcqfc" + s.ContainerName = "nginx" + + s.PodId = util.NewPodId(s.Namespace, s.PodName) + + s.Logger = logp.NewLogger("kubernetes.container") + + s.NodeMetrics = util.NewNodeMetrics() + s.NodeMetrics.CoresAllocatable = util.NewFloat64Metric(2) + s.NodeMetrics.MemoryAllocatable = util.NewFloat64Metric(146227200) + + s.ContainerMetrics = util.NewContainerMetrics() + s.ContainerMetrics.MemoryLimit = util.NewFloat64Metric(14622720) +} +func (s *ContainerTestSuite) ReadTestFile(testFile string) []byte { f, err := os.Open(testFile) - assert.NoError(t, err, "cannot open test file "+testFile) + s.NoError(err, "cannot open test file "+testFile) body, err := ioutil.ReadAll(f) - assert.NoError(t, err, "cannot read test file "+testFile) + s.NoError(err, "cannot read test file "+testFile) - cache := util.NewPerfMetricsCache(120 * time.Second) - cache.NodeCoresAllocatable.Set("gke-beats-default-pool-a5b33e2e-hdww", 2) - cache.NodeMemAllocatable.Set("gke-beats-default-pool-a5b33e2e-hdww", 146227200) - cache.ContainerMemLimit.Set(util.ContainerUID("default", "nginx-deployment-2303442956-pcqfc", "nginx"), 14622720) + return body +} + +func (s *ContainerTestSuite) TestEventMapping() { + s.MetricsRepo.DeleteAllNodeStore() + + s.addNodeMetric(s.NodeMetrics) + s.addContainerMetric(s.ContainerName, s.ContainerMetrics) - events, err := eventMapping(body, cache, logger) - assert.NoError(t, err, "error mapping "+testFile) + body := s.ReadTestFile(testFile) + events, err := eventMapping(body, s.MetricsRepo, s.Logger) - assert.Len(t, events, 1, "got wrong number of events") + s.basicTests(events, err) - testCases := map[string]interface{}{ + cpuMemoryTestCases := map[string]interface{}{ "cpu.usage.core.ns": 43959424, "cpu.usage.nanocores": 11263994, - "logs.available.bytes": int64(98727014400), - "logs.capacity.bytes": int64(101258067968), - "logs.used.bytes": 28672, - "logs.inodes.count": 6258720, - "logs.inodes.free": 6120096, - "logs.inodes.used": 138624, - "memory.available.bytes": 0, "memory.usage.bytes": 1462272, "memory.rss.bytes": 1409024, @@ -78,6 +105,49 @@ func TestEventMapping(t *testing.T) { "memory.usage.node.pct": 0.01, "memory.usage.limit.pct": 0.1, "memory.workingset.limit.pct": 0.09943977591036414, + } + + s.RunMetricsTests(events[0], cpuMemoryTestCases) + + containerEcsFields := ecsfields(events[0], s.Logger) + testEcs := map[string]interface{}{ + "cpu.usage": 0.005631997, + "memory.usage": 0.01, + "name": "nginx", + } + s.RunMetricsTests(containerEcsFields, testEcs) +} + +func (s *ContainerTestSuite) testValue(event mapstr.M, field string, expected interface{}) { + data, err := event.GetValue(field) + s.NoError(err, "Could not read field "+field) + s.EqualValues(expected, data, "Wrong value for field "+field) +} + +func (s *ContainerTestSuite) addContainerMetric(containerName string, containerMetric *util.ContainerMetrics) { + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + podStore, _ := nodeStore.AddPodStore(s.PodId) + containerStore, _ := podStore.AddContainerStore(containerName) + containerStore.SetContainerMetrics(containerMetric) +} + +func (s *ContainerTestSuite) addNodeMetric(nodeMetrics *util.NodeMetrics) { + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + nodeStore.SetNodeMetrics(nodeMetrics) +} + +func (s *ContainerTestSuite) basicTests(events []mapstr.M, err error) { + s.NoError(err, "error mapping "+testFile) + + s.Len(events, 1, "got wrong number of events") + + basicTestCases := map[string]interface{}{ + "logs.available.bytes": int64(98727014400), + "logs.capacity.bytes": int64(101258067968), + "logs.used.bytes": 28672, + "logs.inodes.count": 6258720, + "logs.inodes.free": 6120096, + "logs.inodes.used": 138624, "name": "nginx", @@ -87,23 +157,15 @@ func TestEventMapping(t *testing.T) { "rootfs.inodes.used": 21, } - for k, v := range testCases { - testValue(t, events[0], k, v) - } + s.RunMetricsTests(events[0], basicTestCases) +} - containerEcsFields := ecsfields(events[0], logger) - testEcs := map[string]interface{}{ - "cpu.usage": 0.005631997, - "memory.usage": 0.01, - "name": "nginx", - } - for k, v := range testEcs { - testValue(t, containerEcsFields, k, v) +func (s *ContainerTestSuite) RunMetricsTests(event mapstr.M, testCases map[string]interface{}) { + for k, v := range testCases { + s.testValue(event, k, v) } } -func testValue(t *testing.T, event mapstr.M, field string, value interface{}) { - data, err := event.GetValue(field) - assert.NoError(t, err, "Could not read field "+field) - assert.EqualValues(t, data, value, "Wrong value for field "+field) +func TestContainerTestSuite(t *testing.T) { + suite.Run(t, new(ContainerTestSuite)) } diff --git a/metricbeat/module/kubernetes/container/data.go b/metricbeat/module/kubernetes/container/data.go index 1d83c950970..1fd0f44e85f 100644 --- a/metricbeat/module/kubernetes/container/data.go +++ b/metricbeat/module/kubernetes/container/data.go @@ -29,7 +29,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *logp.Logger) ([]mapstr.M, error) { +func eventMapping(content []byte, metricsRepo *util.MetricsRepo, logger *logp.Logger) ([]mapstr.M, error) { events := []mapstr.M{} var summary kubernetes.Summary @@ -39,9 +39,23 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *lo } node := summary.Node - nodeCores := perfMetrics.NodeCoresAllocatable.Get(node.NodeName) - nodeMem := perfMetrics.NodeMemAllocatable.Get(node.NodeName) + + nodeCores := 0.0 + nodeMem := 0.0 + + nodeStore := metricsRepo.GetNodeStore(node.NodeName) + nodeMetrics := nodeStore.GetNodeMetrics() + if nodeMetrics.CoresAllocatable != nil { + nodeCores = nodeMetrics.CoresAllocatable.Value + } + if nodeMetrics.MemoryAllocatable != nil { + nodeMem = nodeMetrics.MemoryAllocatable.Value + } + for _, pod := range summary.Pods { + podId := util.NewPodId(pod.PodRef.Namespace, pod.PodRef.Name) + podStore := nodeStore.GetPodStore(podId) + for _, container := range pod.Containers { containerEvent := mapstr.M{ mb.ModuleDataKey: mapstr.M{ @@ -127,17 +141,31 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *lo kubernetes2.ShouldPut(containerEvent, "memory.usage.node.pct", float64(container.Memory.UsageBytes)/nodeMem, logger) } - cuid := util.ContainerUID(pod.PodRef.Namespace, pod.PodRef.Name, container.Name) - coresLimit := perfMetrics.ContainerCoresLimit.GetWithDefault(cuid, nodeCores) - memLimit := perfMetrics.ContainerMemLimit.GetWithDefault(cuid, nodeMem) + containerStore := podStore.GetContainerStore(container.Name) + containerMetrics := containerStore.GetContainerMetrics() + + containerCoresLimit := nodeCores + if containerMetrics.CoresLimit != nil { + containerCoresLimit = containerMetrics.CoresLimit.Value + } + + containerMemLimit := nodeMem + if containerMetrics.MemoryLimit != nil { + containerMemLimit = containerMetrics.MemoryLimit.Value + } + + // NOTE: + // we don't currently check if `containerMemLimit` > `nodeMem` as we do in `kubernetes/pod/data.go`. + // There we do check, since if a container doesn't have a limit set, it will inherit the node limits and the sum of all + // the container limits can be greater than the node limits. We assume here the user can set correct limits on containers. - if coresLimit > 0 { - kubernetes2.ShouldPut(containerEvent, "cpu.usage.limit.pct", float64(container.CPU.UsageNanoCores)/1e9/coresLimit, logger) + if containerCoresLimit > 0 { + kubernetes2.ShouldPut(containerEvent, "cpu.usage.limit.pct", float64(container.CPU.UsageNanoCores)/1e9/containerCoresLimit, logger) } - if memLimit > 0 { - kubernetes2.ShouldPut(containerEvent, "memory.usage.limit.pct", float64(container.Memory.UsageBytes)/memLimit, logger) - kubernetes2.ShouldPut(containerEvent, "memory.workingset.limit.pct", float64(container.Memory.WorkingSetBytes)/memLimit, logger) + if containerMemLimit > 0 { + kubernetes2.ShouldPut(containerEvent, "memory.usage.limit.pct", float64(container.Memory.UsageBytes)/containerMemLimit, logger) + kubernetes2.ShouldPut(containerEvent, "memory.workingset.limit.pct", float64(container.Memory.WorkingSetBytes)/containerMemLimit, logger) } events = append(events, containerEvent) diff --git a/metricbeat/module/kubernetes/controllermanager/_meta/data.json b/metricbeat/module/kubernetes/controllermanager/_meta/data.json index 97764da595b..4e1e6227343 100644 --- a/metricbeat/module/kubernetes/controllermanager/_meta/data.json +++ b/metricbeat/module/kubernetes/controllermanager/_meta/data.json @@ -7,31 +7,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.008447322 - } - } + "name": "serviceaccount", + "workqueue": { + "adds": { + "count": 5 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/configmaps?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { diff --git a/metricbeat/module/kubernetes/controllermanager/_meta/fields.yml b/metricbeat/module/kubernetes/controllermanager/_meta/fields.yml index 74f17568ca0..032e1311c9c 100644 --- a/metricbeat/module/kubernetes/controllermanager/_meta/fields.yml +++ b/metricbeat/module/kubernetes/controllermanager/_meta/fields.yml @@ -62,14 +62,14 @@ type: long description: > Number of requests as client - - name: request.duration.sec.bucket.* + - name: request.duration.us.bucket.* type: object object_type: long description: Response latency distribution, histogram buckets - - name: request.duration.sec.sum + - name: request.duration.us.sum type: long - description: Request duration, sum in seconds - - name: request.duration.sec.count + description: Request duration, sum in microseconds + - name: request.duration.us.count type: long description: Request duration, number of operations - name: workqueue @@ -90,16 +90,6 @@ - name: retries.count type: long description: Workqueue number of retries - - name: work.duration.sec.sum - type: long - description: Processing time of workqueue item, sum in seconds - - name: work.duration.sec.count - type: long - description: Processing time of workqueue item, number of operations - - name: work.duration.sec.bucket.* - type: object - object_type: long - description: Processing time of workqueue item, histogram buckets - name: node.collector type: group fields: diff --git a/metricbeat/module/kubernetes/controllermanager/_meta/test/metrics.1.20.expected b/metricbeat/module/kubernetes/controllermanager/_meta/test/metrics.1.20.expected index 12ee9c0e5d1..6e07388c719 100644 --- a/metricbeat/module/kubernetes/controllermanager/_meta/test/metrics.1.20.expected +++ b/metricbeat/module/kubernetes/controllermanager/_meta/test/metrics.1.20.expected @@ -6,27 +6,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 126, + "128000": 7920, + "16000": 7920, + "2000": 1070, + "256000": 7920, + "32000": 7920, + "4000": 3721, + "512000": 7920, + "64000": 7920, + "8000": 7839 }, - "count": 1, - "sum": 0.02086819 + "count": 7921, + "sum": 32028062.373999964 } } } }, - "url": "https://172.18.0.2:6443/api/v1/nodes", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -40,61 +40,6 @@ "Period": 0, "DisableTimeSeries": false }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "name": "DynamicCABundle-csr-controller", - "workqueue": { - "adds": { - "count": 6340 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6340, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 20, - "0.001": 6320, - "0.01": 6340, - "0.1": 6340, - "1": 6340, - "10": 6340 - }, - "count": 6340, - "sum": 2.040141396000006 - } - } - } - } - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, { "RootFields": null, "ModuleFields": null, @@ -102,27 +47,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.009125225 + "sum": 5445.045 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/controllerrevisions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csinodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -143,27 +88,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.432324775 + "sum": 1330.3519999999999 } } } }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/mutatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/resourcequotas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -235,10 +180,51 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "DynamicCABundle-client-ca-bundle", + "client": { + "request": { + "duration": { + "us": { + "bucket": { + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 + }, + "count": 1, + "sum": 6997.829 + } + } + } + }, + "url": "https://172.18.0.2:6443/api/v1/persistentvolumes/%7Bname%7D", + "verb": "GET" + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "name": "endpoint_slice_mirroring", "workqueue": { "adds": { - "count": 1585 + "count": 6 }, "depth": { "count": 0 @@ -251,27 +237,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 1101, - "0.001": 1585, - "0.01": 1585, - "0.1": 1585, - "1": 1585, - "10": 1585 - }, - "count": 1585, - "sum": 0.1301518080000002 - } - } } } }, @@ -293,28 +258,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 47237, - "0.001": 0, - "0.002": 1929, - "0.004": 29430, - "0.008": 46836, - "0.016": 47157, - "0.032": 47188, - "0.064": 47233, - "0.128": 47237, - "0.256": 47237, - "0.512": 47237 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 47237, - "sum": 182.52047819099928 + "count": 1, + "sum": 533260.944 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", - "verb": "PUT" + "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/prioritylevelconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -331,10 +296,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "orphaned_pods_nodes", + "name": "cronjob", "workqueue": { "adds": { - "count": 0 + "count": 8259 }, "depth": { "count": 0 @@ -343,31 +308,10 @@ "sec": 0 }, "retries": { - "count": 0 + "count": 8259 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -386,45 +330,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "horizontalpodautoscaler", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 40, + "128000": 7921, + "16000": 7921, + "2000": 663, + "256000": 7921, + "32000": 7921, + "4000": 3393, + "512000": 7921, + "64000": 7921, + "8000": 7841 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 32966486.31899999 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -444,27 +374,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 31, + "128000": 7921, + "16000": 7920, + "2000": 549, + "256000": 7921, + "32000": 7920, + "4000": 3288, + "512000": 7921, + "64000": 7920, + "8000": 7836 }, - "count": 1, - "sum": 0.005913528 + "count": 7921, + "sum": 33411074.78999993 } } } }, - "url": "https://172.18.0.2:6443/api/v1/services?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -485,27 +415,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 39, - "0.002": 732, - "0.004": 3565, - "0.008": 7856, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 1, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 32.403059004999946 + "count": 1, + "sum": 862.9010000000001 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1/customresourcedefinitions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -519,6 +449,30 @@ "Period": 0, "DisableTimeSeries": false }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "client": { + "request": { + "count": 313 + } + }, + "code": "200", + "host": "172.18.0.2:6443", + "method": "PATCH" + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, { "RootFields": null, "ModuleFields": null, @@ -526,27 +480,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.632319982 + "sum": 5126.341 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingressclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/namespaces?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -567,27 +521,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 19, - "0.002": 510, - "0.004": 3132, - "0.008": 7833, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.78700202399993 + "count": 1, + "sum": 5315.358 } } } }, - "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/autoscaling/v1/horizontalpodautoscalers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -608,27 +562,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 56, - "0.002": 755, - "0.004": 3433, - "0.008": 7830, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 517, + "256000": 7921, + "32000": 7921, + "4000": 3164, + "512000": 7921, + "64000": 7921, + "8000": 7818 }, "count": 7921, - "sum": 32.76269844599994 + "sum": 33747268.298 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -648,12 +602,12 @@ "MetricSetFields": { "client": { "request": { - "count": 313 + "count": 387658 } }, "code": "200", "host": "172.18.0.2:6443", - "method": "PATCH" + "method": "GET" }, "Index": "", "ID": "", @@ -673,27 +627,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 54, + "128000": 7920, + "16000": 7920, + "2000": 733, + "256000": 7920, + "32000": 7920, + "4000": 3497, + "512000": 7921, + "64000": 7920, + "8000": 7848 }, - "count": 1, - "sum": 0.001480236 + "count": 7921, + "sum": 33072480.697999936 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/policy/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -714,27 +668,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 1, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 30, + "128000": 7920, + "16000": 7920, + "2000": 628, + "256000": 7920, + "32000": 7920, + "4000": 3430, + "512000": 7921, + "64000": 7920, + "8000": 7839 }, - "count": 1, - "sum": 0.000862901 + "count": 7921, + "sum": 33331486.811000027 } } } }, - "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1/customresourcedefinitions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/policy/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -752,45 +706,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "resourcequota_primary", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 199746.336 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/replicationcontrollers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -810,27 +750,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 26, + "128000": 7921, + "16000": 7921, + "2000": 576, + "256000": 7921, + "32000": 7921, + "4000": 3377, + "512000": 7921, + "64000": 7921, + "8000": 7840 }, - "count": 1, - "sum": 0.007787689 + "count": 7921, + "sum": 33167423.892999955 } } } }, - "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1/events?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -850,12 +790,29 @@ "MetricSetFields": { "client": { "request": { - "count": 387658 + "duration": { + "us": { + "bucket": { + "+Inf": 7921, + "1000": 17, + "128000": 7921, + "16000": 7921, + "2000": 518, + "256000": 7921, + "32000": 7921, + "4000": 3203, + "512000": 7921, + "64000": 7921, + "8000": 7828 + }, + "count": 7921, + "sum": 33685464.92800006 + } + } } }, - "code": "200", - "host": "172.18.0.2:6443", - "method": "GET" + "url": "https://172.18.0.2:6443/apis/batch/v1beta1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -872,45 +829,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "resourcequota_priority", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1584, + "2000": 1327, + "256000": 1585, + "32000": 1585, + "4000": 1570, + "512000": 1585, + "64000": 1585, + "8000": 1584 }, - "count": 0, - "sum": 0 + "count": 1585, + "sum": 2828044.9830000033 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -930,28 +873,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 2, + "1000": 1, + "128000": 2, + "16000": 2, + "2000": 2, + "256000": 2, + "32000": 2, + "4000": 2, + "512000": 2, + "64000": 2, + "8000": 2 }, - "count": 1, - "sum": 0.005865891 + "count": 2, + "sum": 2408.319 } } } }, - "url": "https://172.18.0.2:6443/api/v1/endpoints?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s", + "verb": "POST" }, "Index": "", "ID": "", @@ -971,27 +914,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 3, + "256000": 3, + "32000": 3, + "4000": 3, + "512000": 3, + "64000": 3, + "8000": 3 }, - "count": 1, - "sum": 0.009297166 + "count": 3, + "sum": 5033.812000000001 } } } }, - "url": "https://172.18.0.2:6443/api/v1/persistentvolumes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D", "verb": "GET" }, "Index": "", @@ -1012,27 +955,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 577, + "128000": 7921, + "16000": 7921, + "2000": 2786, + "256000": 7921, + "32000": 7921, + "4000": 5511, + "512000": 7921, + "64000": 7921, + "8000": 7877 }, - "count": 1, - "sum": 0.011306875 + "count": 7921, + "sum": 24079648.88199997 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterroles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -1050,10 +993,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "serviceaccount", + "name": "ttlcontroller", "workqueue": { "adds": { - "count": 5 + "count": 317 }, "depth": { "count": 0 @@ -1066,27 +1009,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 5, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 4, - "0.00009999999999999999": 5, - "0.001": 5, - "0.01": 5, - "0.1": 5, - "1": 5, - "10": 5 - }, - "count": 5, - "sum": 0.000074208 - } - } } } }, @@ -1105,7 +1027,7 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "token_cleaner", + "name": "replicationmanager", "workqueue": { "adds": { "count": 0 @@ -1121,27 +1043,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -1163,27 +1064,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 18, - "0.002": 561, - "0.004": 3312, - "0.008": 7839, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.337573244000026 + "count": 1, + "sum": 4378.482 } } } }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csidrivers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -1204,28 +1105,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 18103, - "0.001": 0, - "0.002": 2459, - "0.004": 14393, - "0.008": 16165, - "0.016": 17812, - "0.032": 18088, - "0.064": 18102, - "0.128": 18103, - "0.256": 18103, - "0.512": 18103 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 18103, - "sum": 70.20654167600028 + "count": 1, + "sum": 6075.867 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/roles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -1242,10 +1143,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "ClusterRoleAggregator", + "name": "DynamicCABundle-request-header", "workqueue": { "adds": { - "count": 27 + "count": 1585 }, "depth": { "count": 0 @@ -1258,27 +1159,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 27, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 5, - "0.001": 27, - "0.01": 27, - "0.1": 27, - "1": 27, - "10": 27 - }, - "count": 27, - "sum": 0.0066875810000000015 - } - } } } }, @@ -1297,42 +1177,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "volumes", - "workqueue": { - "adds": { - "count": 6340 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 6340, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 5380, - "0.001": 6340, - "0.01": 6340, - "0.1": 6340, - "1": 6340, - "10": 6340 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 6340, - "sum": 0.4856128449999997 + "count": 1, + "sum": 5955 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/batch/v1/cronjobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -1352,27 +1221,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 52, + "128000": 7920, + "16000": 7920, + "2000": 739, + "256000": 7920, + "32000": 7920, + "4000": 3489, + "512000": 7921, + "64000": 7920, + "8000": 7832 }, - "count": 1, - "sum": 0.009554271 + "count": 7921, + "sum": 32978230.279000036 } } } }, - "url": "https://172.18.0.2:6443/apis/policy/v1/poddisruptionbudgets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/extensions/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -1393,27 +1262,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.004378482 + "sum": 383570.902 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csidrivers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -1433,29 +1302,12 @@ "MetricSetFields": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 54, - "0.002": 733, - "0.004": 3497, - "0.008": 7848, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.072480697999936 - } - } + "count": 6513 } }, - "url": "https://172.18.0.2:6443/apis/policy/v1?timeout=32s", - "verb": "GET" + "code": "201", + "host": "172.18.0.2:6443", + "method": "POST" }, "Index": "", "ID": "", @@ -1475,28 +1327,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 24, - "0.002": 574, - "0.004": 3334, - "0.008": 7842, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 315, + "1000": 0, + "128000": 315, + "16000": 315, + "2000": 2, + "256000": 315, + "32000": 315, + "4000": 178, + "512000": 315, + "64000": 315, + "8000": 313 }, - "count": 7921, - "sum": 33.281374038000045 + "count": 315, + "sum": 1259640.6289999997 } } } }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events/%7Bname%7D", + "verb": "PATCH" }, "Index": "", "ID": "", @@ -1513,10 +1365,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "claims", + "name": "deployment", "workqueue": { "adds": { - "count": 6340 + "count": 205 }, "depth": { "count": 0 @@ -1524,31 +1376,54 @@ "longestrunning": { "sec": 0 }, + "retries": { + "count": 198 + }, "unfinished": { "sec": 0 - }, - "work": { + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 6340, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 3961, - "0.001": 6335, - "0.01": 6340, - "0.1": 6340, - "1": 6340, - "10": 6340 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 6340, - "sum": 0.6305873910000022 + "count": 1, + "sum": 1257.167 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/serviceaccounts?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -1568,28 +1443,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 18, - "0.002": 524, - "0.004": 3158, - "0.008": 7831, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 3200, + "1000": 0, + "128000": 3200, + "16000": 3196, + "2000": 190, + "256000": 3200, + "32000": 3199, + "4000": 2389, + "512000": 3200, + "64000": 3200, + "8000": 3174 }, - "count": 7921, - "sum": 33.75089559099998 + "count": 3200, + "sum": 11213909.066999974 } } } }, - "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events", + "verb": "POST" }, "Index": "", "ID": "", @@ -1606,10 +1481,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "DynamicCABundle-request-header", + "name": "endpoint_slice", "workqueue": { "adds": { - "count": 1585 + "count": 10 }, "depth": { "count": 0 @@ -1618,31 +1493,10 @@ "sec": 0 }, "retries": { - "count": 0 + "count": 7 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 1140, - "0.001": 1585, - "0.01": 1585, - "0.1": 1585, - "1": 1585, - "10": 1585 - }, - "count": 1585, - "sum": 0.128010803 - } - } } } }, @@ -1664,28 +1518,62 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1582, + "2000": 0, + "256000": 1585, + "32000": 1585, + "4000": 688, + "512000": 1585, + "64000": 1585, + "8000": 1569 }, - "count": 1, - "sum": 0.051489332 + "count": 1585, + "sum": 6896629.352000009 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/leases?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", + "verb": "DELETE" + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "name": "resource_quota_controller_resource_changes", + "workqueue": { + "adds": { + "count": 13983 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 + } + } }, "Index": "", "ID": "", @@ -1705,27 +1593,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 28, + "128000": 7920, + "16000": 7920, + "2000": 617, + "256000": 7921, + "32000": 7920, + "4000": 3327, + "512000": 7921, + "64000": 7920, + "8000": 7832 }, - "count": 1, - "sum": 0.010918125 + "count": 7921, + "sum": 33451801.867 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/daemonsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -1743,10 +1631,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "bootstrap_signer_queue", + "name": "root_ca_cert_publisher", "workqueue": { "adds": { - "count": 1 + "count": 10 }, "depth": { "count": 0 @@ -1759,27 +1647,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 1, - "0.01": 1, - "0.1": 1, - "1": 1, - "10": 1 - }, - "count": 1, - "sum": 0.000200496 - } - } } } }, @@ -1800,12 +1667,29 @@ "MetricSetFields": { "client": { "request": { - "count": 6513 + "duration": { + "us": { + "bucket": { + "+Inf": 2, + "1000": 0, + "128000": 2, + "16000": 2, + "2000": 0, + "256000": 2, + "32000": 2, + "4000": 1, + "512000": 2, + "64000": 2, + "8000": 1 + }, + "count": 2, + "sum": 12623.51 + } + } } }, - "code": "201", - "host": "172.18.0.2:6443", - "method": "POST" + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D/status", + "verb": "PUT" }, "Index": "", "ID": "", @@ -1825,27 +1709,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 1, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.101172363 + "sum": 903.169 } } } }, - "url": "https://172.18.0.2:6443/api/v1/pods?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/podtemplates?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -1866,27 +1750,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.005936288 + "sum": 2144.357 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1/csistoragecapacities?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/secrets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -1907,27 +1791,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.334037013 + "sum": 5913.528 } } } }, - "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1/priorityclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/services?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -1945,31 +1829,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 21, - "0.002": 536, - "0.004": 3200, - "0.008": 7830, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.739958584 - } - } + "name": "garbage_collector_attempt_to_delete", + "workqueue": { + "adds": { + "count": 1592 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1beta1?timeout=32s", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -1986,31 +1863,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.006997829 - } - } + "name": "pvcs", + "workqueue": { + "adds": { + "count": 2 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/persistentvolumes/%7Bname%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -2027,10 +1897,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "noexec_taint_node", + "name": "daemonset", "workqueue": { "adds": { - "count": 1 + "count": 6 }, "depth": { "count": 0 @@ -2038,31 +1908,54 @@ "longestrunning": { "sec": 0 }, + "retries": { + "count": 0 + }, "unfinished": { "sec": 0 - }, - "work": { + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 1, - "0.001": 1, - "0.01": 1, - "0.1": 1, - "1": 1, - "10": 1 + "+Inf": 7921, + "1000": 18, + "128000": 7921, + "16000": 7921, + "2000": 524, + "256000": 7921, + "32000": 7921, + "4000": 3158, + "512000": 7921, + "64000": 7921, + "8000": 7831 }, - "count": 1, - "sum": 0.000021587 + "count": 7921, + "sum": 33750895.59099998 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -2082,27 +1975,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.001330352 + "sum": 7787.689 } } } }, - "url": "https://172.18.0.2:6443/api/v1/resourcequotas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1/events?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -2120,7 +2013,7 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "ephemeral_volume", + "name": "disruption_recheck", "workqueue": { "adds": { "count": 0 @@ -2136,27 +2029,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -2178,27 +2050,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 28, - "0.002": 617, - "0.004": 3327, - "0.008": 7832, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7921, - "0.512": 7921 + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 556, + "256000": 7921, + "32000": 7921, + "4000": 3303, + "512000": 7921, + "64000": 7921, + "8000": 7838 }, "count": 7921, - "sum": 33.451801867 + "sum": 33369705.887999758 } } } }, - "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2216,10 +2088,34 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "pvcprotection", + "client": { + "request": { + "count": 1585 + } + }, + "code": "409", + "host": "172.18.0.2:6443", + "method": "PUT" + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "name": "garbage_collector_graph_changes", "workqueue": { "adds": { - "count": 0 + "count": 190259 }, "depth": { "count": 0 @@ -2232,27 +2128,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -2271,31 +2146,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 52, - "0.002": 739, - "0.004": 3489, - "0.008": 7832, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 - }, - "count": 7921, - "sum": 32.97823027900004 - } - } + "name": "disruption", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/extensions/v1beta1?timeout=32s", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -2315,28 +2183,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 2, - "sum": 0.012623510000000001 + "count": 1, + "sum": 734162.309 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/apis/policy/v1beta1/podsecuritypolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -2356,27 +2224,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "+Inf": 7921, + "1000": 21, + "128000": 7921, + "16000": 7921, + "2000": 536, + "256000": 7921, + "32000": 7921, + "4000": 3200, + "512000": 7921, + "64000": 7921, + "8000": 7830 }, - "count": 1, - "sum": 0.48271115 + "count": 7921, + "sum": 33739958.584 } } } }, - "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1/runtimeclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2397,27 +2265,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.001286213 + "sum": 632319.982 } } } }, - "url": "https://172.18.0.2:6443/healthz?timeout=32s", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingressclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -2438,27 +2306,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.010220903 + "sum": 1286.213 } } } }, - "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/healthz?timeout=32s", "verb": "GET" }, "Index": "", @@ -2479,27 +2347,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 3, - "0.004": 3, - "0.008": 3, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 7921, + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 565, + "256000": 7921, + "32000": 7921, + "4000": 3309, + "512000": 7921, + "64000": 7921, + "8000": 7819 }, - "count": 3, - "sum": 0.005033812 + "count": 7921, + "sum": 33406713.9199999 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2519,12 +2387,29 @@ "MetricSetFields": { "client": { "request": { - "count": 1585 + "duration": { + "us": { + "bucket": { + "+Inf": 2, + "1000": 0, + "128000": 2, + "16000": 2, + "2000": 0, + "256000": 2, + "32000": 2, + "4000": 0, + "512000": 2, + "64000": 2, + "8000": 1 + }, + "count": 2, + "sum": 19134.250999999997 + } + } } }, - "code": "409", - "host": "172.18.0.2:6443", - "method": "PUT" + "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1/tokenreviews", + "verb": "POST" }, "Index": "", "ID": "", @@ -2544,28 +2429,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3200, - "0.001": 0, - "0.002": 190, - "0.004": 2389, - "0.008": 3174, - "0.016": 3196, - "0.032": 3199, - "0.064": 3200, - "0.128": 3200, - "0.256": 3200, - "0.512": 3200 + "+Inf": 7921, + "1000": 64, + "128000": 7921, + "16000": 7921, + "2000": 876, + "256000": 7921, + "32000": 7921, + "4000": 3720, + "512000": 7921, + "64000": 7921, + "8000": 7848 }, - "count": 3200, - "sum": 11.213909066999973 + "count": 7921, + "sum": 32026618.21400003 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events", - "verb": "POST" + "url": "https://172.18.0.2:6443/api/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -2582,31 +2467,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 - }, - "count": 1, - "sum": 0.533260944 - } - } + "name": "certificate", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/prioritylevelconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -2626,27 +2504,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 23, - "0.002": 556, - "0.004": 3303, - "0.008": 7838, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 22, + "128000": 7921, + "16000": 7921, + "2000": 552, + "256000": 7921, + "32000": 7921, + "4000": 3260, + "512000": 7921, + "64000": 7921, + "8000": 7839 }, "count": 7921, - "sum": 33.36970588799976 + "sum": 33459086.083999977 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/batch/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2667,27 +2545,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 30, + "128000": 7920, + "16000": 7920, + "2000": 512, + "256000": 7921, + "32000": 7920, + "4000": 3208, + "512000": 7921, + "64000": 7920, + "8000": 7823 }, - "count": 1, - "sum": 0.009219587 + "count": 7921, + "sum": 33784837.885 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/networkpolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2708,27 +2586,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 26, - "0.002": 576, - "0.004": 3377, - "0.008": 7840, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 33, + "128000": 7921, + "16000": 7921, + "2000": 503, + "256000": 7921, + "32000": 7921, + "4000": 3059, + "512000": 7921, + "64000": 7921, + "8000": 7828 }, "count": 7921, - "sum": 33.167423892999956 + "sum": 33942666.33799993 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2746,72 +2624,21 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.008323121 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/endpointslices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.005635227 - } - } + "name": "volumes", + "workqueue": { + "adds": { + "count": 6340 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/apps/v1/statefulsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -2828,10 +2655,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "certificate", + "name": "DynamicServingCertificateController", "workqueue": { "adds": { - "count": 0 + "count": 1585 }, "depth": { "count": 0 @@ -2844,27 +2671,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -2886,27 +2692,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 34, - "0.002": 647, - "0.004": 3450, - "0.008": 7840, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 31, + "128000": 7921, + "16000": 7921, + "2000": 596, + "256000": 7921, + "32000": 7921, + "4000": 3350, + "512000": 7921, + "64000": 7921, + "8000": 7826 }, "count": 7921, - "sum": 32.91060117799999 + "sum": 33256375.38899997 } } } }, - "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -2924,10 +2730,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "pvcs", + "name": "endpoint", "workqueue": { "adds": { - "count": 2 + "count": 7 }, "depth": { "count": 0 @@ -2936,31 +2742,10 @@ "sec": 0 }, "retries": { - "count": 0 + "count": 4 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 2, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 2, - "0.001": 2, - "0.01": 2, - "0.1": 2, - "1": 2, - "10": 2 - }, - "count": 2, - "sum": 0.000036973 - } - } } } }, @@ -2982,27 +2767,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 2, - "0.004": 3, - "0.008": 3, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 47243, + "1000": 0, + "128000": 47241, + "16000": 47143, + "2000": 433, + "256000": 47241, + "32000": 47174, + "4000": 3292, + "512000": 47241, + "64000": 47240, + "8000": 21617 }, - "count": 3, - "sum": 0.004963458 + "count": 47243, + "sum": 379170587.88699937 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", "verb": "GET" }, "Index": "", @@ -3020,45 +2805,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "deployment", - "workqueue": { - "adds": { - "count": 205 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 198 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 205, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 167, - "0.01": 205, - "0.1": 205, - "1": 205, - "10": 205 + "+Inf": 7921, + "1000": 34, + "128000": 7921, + "16000": 7921, + "2000": 647, + "256000": 7921, + "32000": 7921, + "4000": 3450, + "512000": 7921, + "64000": 7921, + "8000": 7840 }, - "count": 205, - "sum": 0.152872125 + "count": 7921, + "sum": 32910601.177999992 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1beta1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -3075,10 +2846,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "garbage_collector_graph_changes", + "name": "replicaset", "workqueue": { "adds": { - "count": 190259 + "count": 9 }, "depth": { "count": 0 @@ -3091,27 +2862,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 190259, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 312, - "0.000009999999999999999": 162187, - "0.00009999999999999999": 189930, - "0.001": 190257, - "0.01": 190259, - "0.1": 190259, - "1": 190259, - "10": 190259 - }, - "count": 190259, - "sum": 1.5498176499999963 - } - } } } }, @@ -3133,27 +2883,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 1, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.000903169 + "sum": 6161.415 } } } }, - "url": "https://172.18.0.2:6443/api/v1/podtemplates?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/volumeattachments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -3171,45 +2921,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "disruption", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 20, + "128000": 7921, + "16000": 7921, + "2000": 547, + "256000": 7921, + "32000": 7921, + "4000": 3182, + "512000": 7921, + "64000": 7921, + "8000": 7836 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 33695198.23299993 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1beta1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -3229,28 +2965,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "+Inf": 2, + "1000": 0, + "128000": 2, + "16000": 2, + "2000": 0, + "256000": 2, + "32000": 2, + "4000": 1, + "512000": 2, + "64000": 2, + "8000": 1 }, - "count": 1, - "sum": 0.734162309 + "count": 2, + "sum": 16458.466 } } } }, - "url": "https://172.18.0.2:6443/apis/policy/v1beta1/podsecuritypolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D/status", + "verb": "PUT" }, "Index": "", "ID": "", @@ -3267,65 +3003,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "replicaset", - "workqueue": { - "adds": { - "count": 9 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 9, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 3, - "0.001": 7, - "0.01": 8, - "0.1": 9, - "1": 9, - "10": 9 + "+Inf": 139, + "1000": 0, + "128000": 139, + "16000": 138, + "2000": 0, + "256000": 139, + "32000": 139, + "4000": 29, + "512000": 139, + "64000": 139, + "8000": 138 }, - "count": 9, - "sum": 0.020865991 + "count": 139, + "sum": 676824.807 } } } - } - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "leader": { - "is_master": true }, - "name": "kube-controller-manager" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D/token", + "verb": "POST" }, "Index": "", "ID": "", @@ -3345,28 +3047,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 47237, + "1000": 0, + "128000": 47237, + "16000": 47157, + "2000": 1929, + "256000": 47237, + "32000": 47188, + "4000": 29430, + "512000": 47237, + "64000": 47233, + "8000": 46836 }, - "count": 1, - "sum": 0.007561396 + "count": 47237, + "sum": 182520478.1909993 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/rolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", + "verb": "PUT" }, "Index": "", "ID": "", @@ -3386,28 +3088,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 20, - "0.002": 547, - "0.004": 3182, - "0.008": 7836, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 0, + "256000": 3, + "32000": 3, + "4000": 0, + "512000": 3, + "64000": 3, + "8000": 2 }, - "count": 7921, - "sum": 33.69519823299993 + "count": 3, + "sum": 20303.007 } } } }, - "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1beta1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/namespaces/%7Bnamespace%7D/endpointslices/%7Bname%7D", + "verb": "PUT" }, "Index": "", "ID": "", @@ -3424,31 +3126,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 - }, - "count": 1, - "sum": 0.383570902 - } - } - } + "leader": { + "is_master": true }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "name": "kube-controller-manager" }, "Index": "", "ID": "", @@ -3468,27 +3149,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 29, + "128000": 7921, + "16000": 7921, + "2000": 575, + "256000": 7921, + "32000": 7921, + "4000": 3324, + "512000": 7921, + "64000": 7921, + "8000": 7833 }, - "count": 1, - "sum": 0.001325518 + "count": 7921, + "sum": 33364429.504999924 } } } }, - "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1/apiservices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -3509,27 +3190,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 41, - "0.002": 751, - "0.004": 3563, - "0.008": 7836, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 56, + "128000": 7921, + "16000": 7921, + "2000": 755, + "256000": 7921, + "32000": 7921, + "4000": 3433, + "512000": 7921, + "64000": 7921, + "8000": 7830 }, "count": 7921, - "sum": 32.45968532300009 + "sum": 32762698.44599994 } } } }, - "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -3550,27 +3231,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.006885754 + "sum": 1480.236 } } } }, - "url": "https://172.18.0.2:6443/apis/extensions/v1beta1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -3588,10 +3269,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "resource_quota_controller_resource_changes", + "name": "statefulset", "workqueue": { "adds": { - "count": 13983 + "count": 2 }, "depth": { "count": 0 @@ -3604,82 +3285,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 13983, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 82, - "0.000009999999999999999": 10299, - "0.00009999999999999999": 13968, - "0.001": 13983, - "0.01": 13983, - "0.1": 13983, - "1": 13983, - "10": 13983 - }, - "count": 13983, - "sum": 0.10729375299999992 - } - } - } - } - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "name": "serviceaccount_tokens_secret", - "workqueue": { - "adds": { - "count": 45 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 45, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 26, - "0.00009999999999999999": 29, - "0.001": 36, - "0.01": 45, - "0.1": 45, - "1": 45, - "10": 45 - }, - "count": 45, - "sum": 0.015241941000000002 - } - } } } }, @@ -3701,27 +3306,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.003386321 + "sum": 5936.288 } } } }, - "url": "https://172.18.0.2:6443/api/v1/nodes/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1/csistoragecapacities?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -3742,27 +3347,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 40, - "0.002": 663, - "0.004": 3393, - "0.008": 7841, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 27, + "128000": 7921, + "16000": 7921, + "2000": 621, + "256000": 7921, + "32000": 7921, + "4000": 3345, + "512000": 7921, + "64000": 7921, + "8000": 7858 }, "count": 7921, - "sum": 32.96648631899999 + "sum": 33030734.36400006 } } } }, - "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -3783,27 +3388,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 27, - "0.002": 569, - "0.004": 3377, - "0.008": 7832, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 33.49167660099996 + "count": 1, + "sum": 10220.903 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -3824,27 +3429,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 3440, + "1000": 3, + "128000": 3440, + "16000": 3436, + "2000": 649, + "256000": 3440, + "32000": 3439, + "4000": 3006, + "512000": 3440, + "64000": 3440, + "8000": 3373 }, - "count": 1, - "sum": 0.001206621 + "count": 3440, + "sum": 10386672.363999987 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/storageclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", "verb": "GET" }, "Index": "", @@ -3862,31 +3467,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.006161415 - } - } + "name": "ttl_jobs_to_delete", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/volumeattachments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -3906,28 +3504,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 23, - "0.002": 565, - "0.004": 3309, - "0.008": 7819, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 0, + "256000": 3, + "32000": 3, + "4000": 0, + "512000": 3, + "64000": 3, + "8000": 2 }, - "count": 7921, - "sum": 33.4067139199999 + "count": 3, + "sum": 20853.000999999997 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/endpoints/%7Bname%7D", + "verb": "PUT" }, "Index": "", "ID": "", @@ -3947,27 +3545,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 126, - "0.002": 1070, - "0.004": 3721, - "0.008": 7839, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7920 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 7921, - "sum": 32.028062373999965 + "count": 1, + "sum": 584920.7239999999 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -3988,27 +3586,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 27, - "0.002": 621, - "0.004": 3345, - "0.008": 7858, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.03073436400006 + "count": 1, + "sum": 6118.409 } } } }, - "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apps/v1/deployments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -4029,27 +3627,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 26, - "0.002": 532, - "0.004": 3205, - "0.008": 7835, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.847457393999925 + "count": 1, + "sum": 5635.227 } } } }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apps/v1/statefulsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -4070,27 +3668,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 17, - "0.002": 518, - "0.004": 3203, - "0.008": 7828, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 27, + "128000": 7920, + "16000": 7920, + "2000": 569, + "256000": 7920, + "32000": 7920, + "4000": 3377, + "512000": 7921, + "64000": 7920, + "8000": 7832 }, "count": 7921, - "sum": 33.685464928000066 + "sum": 33491676.60099996 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -4111,28 +3709,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 315, - "0.001": 0, - "0.002": 2, - "0.004": 178, - "0.008": 313, - "0.016": 315, - "0.032": 315, - "0.064": 315, - "0.128": 315, - "0.256": 315, - "0.512": 315 + "+Inf": 3437, + "1000": 0, + "128000": 3437, + "16000": 3435, + "2000": 183, + "256000": 3437, + "32000": 3435, + "4000": 3001, + "512000": 3437, + "64000": 3437, + "8000": 3270 }, - "count": 315, - "sum": 1.2596406289999997 + "count": 3437, + "sum": 11231488.972000027 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events/%7Bname%7D", - "verb": "PATCH" + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D/status", + "verb": "PUT" }, "Index": "", "ID": "", @@ -4152,28 +3750,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 139, - "0.001": 0, - "0.002": 0, - "0.004": 29, - "0.008": 138, - "0.016": 138, - "0.032": 139, - "0.064": 139, - "0.128": 139, - "0.256": 139, - "0.512": 139 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 139, - "sum": 0.676824807 + "count": 1, + "sum": 9297.166000000001 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D/token", - "verb": "POST" + "url": "https://172.18.0.2:6443/api/v1/persistentvolumes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -4193,27 +3791,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 39, - "0.002": 652, - "0.004": 3406, - "0.008": 7844, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 32.92865359800001 + "count": 1, + "sum": 9125.225 } } } }, - "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apps/v1/controllerrevisions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -4233,29 +3831,12 @@ "MetricSetFields": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 30, - "0.002": 512, - "0.004": 3208, - "0.008": 7823, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.784837884999995 - } - } + "count": 3170 } }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1?timeout=32s", - "verb": "GET" + "code": "200", + "host": "172.18.0.2:6443", + "method": "DELETE" }, "Index": "", "ID": "", @@ -4272,7 +3853,7 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "DynamicServingCertificateController", + "name": "DynamicCABundle-client-ca-bundle", "workqueue": { "adds": { "count": 1585 @@ -4288,27 +3869,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 4, - "0.00009999999999999999": 1565, - "0.001": 1585, - "0.01": 1585, - "0.1": 1585, - "1": 1585, - "10": 1585 - }, - "count": 1585, - "sum": 0.04380364099999996 - } - } } } }, @@ -4329,12 +3889,29 @@ "MetricSetFields": { "client": { "request": { - "count": 3170 + "duration": { + "us": { + "bucket": { + "+Inf": 7921, + "1000": 21, + "128000": 7921, + "16000": 7920, + "2000": 480, + "256000": 7921, + "32000": 7920, + "4000": 3157, + "512000": 7921, + "64000": 7921, + "8000": 7835 + }, + "count": 7921, + "sum": 33900589.35399998 + } + } } }, - "code": "200", - "host": "172.18.0.2:6443", - "method": "DELETE" + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -4351,45 +3928,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "endpoint", - "workqueue": { - "adds": { - "count": 7 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 4 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 2, - "0.001": 4, - "0.01": 7, - "0.1": 7, - "1": 7, - "10": 7 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7, - "sum": 0.022435438000000002 + "count": 1, + "sum": 1206.6209999999999 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/storageclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -4409,27 +3972,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.584920724 + "sum": 3386.321 } } } }, - "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/nodes/%7Bname%7D", "verb": "GET" }, "Index": "", @@ -4450,27 +4013,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 21, - "0.002": 480, - "0.004": 3157, - "0.008": 7835, - "0.016": 7920, - "0.032": 7920, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 24, + "128000": 7921, + "16000": 7921, + "2000": 574, + "256000": 7921, + "32000": 7921, + "4000": 3334, + "512000": 7921, + "64000": 7921, + "8000": 7842 }, "count": 7921, - "sum": 33.900589353999976 + "sum": 33281374.038000047 } } } }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/autoscaling/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -4484,6 +4047,40 @@ "Period": 0, "DisableTimeSeries": false }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "name": "job", + "workqueue": { + "adds": { + "count": 9938 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 12734 + }, + "unfinished": { + "sec": 0 + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, { "RootFields": null, "ModuleFields": null, @@ -4491,28 +4088,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.001": 1, - "0.002": 2, - "0.004": 2, - "0.008": 2, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 + "+Inf": 7921, + "1000": 6878, + "128000": 7921, + "16000": 7915, + "2000": 7793, + "256000": 7921, + "32000": 7915, + "4000": 7886, + "512000": 7921, + "64000": 7921, + "8000": 7915 }, - "count": 2, - "sum": 0.002408319 + "count": 7921, + "sum": 6718848.776000002 } } } }, - "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s", - "verb": "POST" + "url": "https://172.18.0.2:6443/apis?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -4532,28 +4129,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 2, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 3, - "sum": 0.020853000999999996 + "count": 1, + "sum": 1479.616 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/endpoints/%7Bname%7D", - "verb": "PUT" + "url": "https://172.18.0.2:6443/api/v1/nodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -4570,10 +4167,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "cronjob", + "name": "ephemeral_volume", "workqueue": { "adds": { - "count": 8259 + "count": 0 }, "depth": { "count": 0 @@ -4582,31 +4179,10 @@ "sec": 0 }, "retries": { - "count": 8259 + "count": 0 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 8259, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 5664, - "0.1": 8258, - "1": 8259, - "10": 8259 - }, - "count": 8259, - "sum": 82.251850376 - } - } } } }, @@ -4628,28 +4204,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1585, + "2000": 33, + "256000": 1585, + "32000": 1585, + "4000": 1552, + "512000": 1585, + "64000": 1585, + "8000": 1585 }, - "count": 1, - "sum": 0.006075867 + "count": 1585, + "sum": 3998630.6189999986 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/roles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", + "verb": "DELETE" }, "Index": "", "ID": "", @@ -4669,27 +4245,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 19, + "128000": 7921, + "16000": 7921, + "2000": 510, + "256000": 7921, + "32000": 7921, + "4000": 3132, + "512000": 7921, + "64000": 7921, + "8000": 7833 }, - "count": 1, - "sum": 0.001479616 + "count": 7921, + "sum": 33787002.02399993 } } } }, - "url": "https://172.18.0.2:6443/api/v1/nodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -4710,27 +4286,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 39, + "128000": 7921, + "16000": 7921, + "2000": 732, + "256000": 7921, + "32000": 7921, + "4000": 3565, + "512000": 7921, + "64000": 7921, + "8000": 7856 }, - "count": 1, - "sum": 0.006118409 + "count": 7921, + "sum": 32403059.004999947 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/deployments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -4751,27 +4327,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.008447322 + "sum": 11306.875 } } } }, - "url": "https://172.18.0.2:6443/api/v1/configmaps?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterroles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -4789,10 +4365,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "namespace", + "name": "ClusterRoleAggregator", "workqueue": { "adds": { - "count": 0 + "count": 27 }, "depth": { "count": 0 @@ -4805,27 +4381,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -4844,10 +4399,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "volume_expand", + "name": "DynamicCABundle-csr-controller", "workqueue": { "adds": { - "count": 1 + "count": 6340 }, "depth": { "count": 0 @@ -4860,27 +4415,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 1, - "0.1": 1, - "1": 1, - "10": 1 - }, - "count": 1, - "sum": 0.007196112 - } - } } } }, @@ -4902,27 +4436,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 1361, - "0.002": 1675, - "0.004": 2198, - "0.008": 3799, - "0.016": 7486, - "0.032": 7907, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 24, + "128000": 7921, + "16000": 7921, + "2000": 551, + "256000": 7921, + "32000": 7921, + "4000": 3241, + "512000": 7921, + "64000": 7921, + "8000": 7821 }, "count": 7921, - "sum": 64.48029232100004 + "sum": 33549185.952999942 } } } }, - "url": "https://172.18.0.2:6443/api?timeout=32s", + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1?timeout=32s", "verb": "GET" }, "Index": "", @@ -4943,27 +4477,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.002144357 + "sum": 482711.14999999997 } } } }, - "url": "https://172.18.0.2:6443/api/v1/secrets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1/runtimeclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -4981,45 +4515,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "statefulset", - "workqueue": { - "adds": { - "count": 2 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 1, - "0.01": 2, - "0.1": 2, - "1": 2, - "10": 2 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 2, - "sum": 0.002338638 + "count": 1, + "sum": 4304.235 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/limitranges?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -5039,27 +4559,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 59, - "0.002": 864, - "0.004": 3725, - "0.008": 7849, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 7921, - "sum": 31.881754901999994 + "count": 1, + "sum": 101172.363 } } } }, - "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/pods?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -5077,45 +4597,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "endpoint_slice", - "workqueue": { - "adds": { - "count": 10 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 7 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 10, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 2, - "0.001": 7, - "0.01": 10, - "0.1": 10, - "1": 10, - "10": 10 + "+Inf": 18103, + "1000": 0, + "128000": 18103, + "16000": 17812, + "2000": 2459, + "256000": 18103, + "32000": 18088, + "4000": 14393, + "512000": 18103, + "64000": 18102, + "8000": 16165 }, - "count": 10, - "sum": 0.022632885 + "count": 18103, + "sum": 70206541.67600028 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D/status", + "verb": "PUT" }, "Index": "", "ID": "", @@ -5135,28 +4641,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 2, - "sum": 0.016458466 + "count": 1, + "sum": 334037.013 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1/priorityclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -5173,31 +4679,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 29, - "0.002": 575, - "0.004": 3324, - "0.008": 7833, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.364429504999926 - } - } + "name": "garbage_collector_attempt_to_orphan", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1beta1?timeout=32s", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -5214,7 +4713,7 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "ttl_jobs_to_delete", + "name": "horizontalpodautoscaler", "workqueue": { "adds": { "count": 0 @@ -5230,29 +4729,49 @@ }, "unfinished": { "sec": 0 - }, - "work": { + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 20868.19 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/nodes", + "verb": "GET" }, "Index": "", "ID": "", @@ -5272,27 +4791,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.001257167 + "sum": 432324.775 } } } }, - "url": "https://172.18.0.2:6443/api/v1/serviceaccounts?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/mutatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -5310,7 +4829,7 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "pvprotection", + "name": "namespace", "workqueue": { "adds": { "count": 0 @@ -5326,27 +4845,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -5368,27 +4866,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.005126341 + "sum": 684430.812 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterrolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -5406,10 +4904,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "daemonset", + "name": "serviceaccount", "workqueue": { "adds": { - "count": 6 + "count": 5 }, "depth": { "count": 0 @@ -5422,27 +4920,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 4, - "0.01": 6, - "0.1": 6, - "1": 6, - "10": 6 - }, - "count": 6, - "sum": 0.005056306000000001 - } - } } } }, @@ -5464,68 +4941,27 @@ "client": { "request": { "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 577, - "0.002": 2786, - "0.004": 5511, - "0.008": 7877, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 24.07964888199997 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1?timeout=32s", - "verb": "GET" - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.005955 + "sum": 1325.518 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/cronjobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1/apiservices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -5546,27 +4982,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 0, + "256000": 3, + "32000": 3, + "4000": 3, + "512000": 3, + "64000": 3, + "8000": 3 }, - "count": 1, - "sum": 0.298650755 + "count": 3, + "sum": 9478.231 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/jobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D", "verb": "GET" }, "Index": "", @@ -5584,10 +5020,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "root_ca_cert_publisher", + "name": "noexec_taint_pod", "workqueue": { "adds": { - "count": 10 + "count": 4770 }, "depth": { "count": 0 @@ -5595,32 +5031,8 @@ "longestrunning": { "sec": 0 }, - "retries": { - "count": 0 - }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 10, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 2, - "0.00009999999999999999": 10, - "0.001": 10, - "0.01": 10, - "0.1": 10, - "1": 10, - "10": 10 - }, - "count": 10, - "sum": 0.00032599999999999996 - } - } } } }, @@ -5642,27 +5054,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 23, - "0.002": 517, - "0.004": 3164, - "0.008": 7818, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 2, + "256000": 3, + "32000": 3, + "4000": 3, + "512000": 3, + "64000": 3, + "8000": 3 }, - "count": 7921, - "sum": 33.747268298 + "count": 3, + "sum": 4963.458 } } } }, - "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D", "verb": "GET" }, "Index": "", @@ -5683,27 +5095,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 31, - "0.002": 549, - "0.004": 3288, - "0.008": 7836, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.41107478999993 + "count": 1, + "sum": 7561.396 } } } }, - "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/rolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -5724,68 +5136,27 @@ "client": { "request": { "duration": { - "sec": { - "bucket": { - "+Inf": 2, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 - }, - "count": 2, - "sum": 0.019134250999999998 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1/tokenreviews", - "verb": "POST" - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3440, - "0.001": 3, - "0.002": 649, - "0.004": 3006, - "0.008": 3373, - "0.016": 3436, - "0.032": 3439, - "0.064": 3440, - "0.128": 3440, - "0.256": 3440, - "0.512": 3440 + "+Inf": 7921, + "1000": 1361, + "128000": 7921, + "16000": 7486, + "2000": 1675, + "256000": 7921, + "32000": 7907, + "4000": 2198, + "512000": 7921, + "64000": 7921, + "8000": 3799 }, - "count": 3440, - "sum": 10.386672363999987 + "count": 7921, + "sum": 64480292.32100003 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", + "url": "https://172.18.0.2:6443/api?timeout=32s", "verb": "GET" }, "Index": "", @@ -5806,28 +5177,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 0, - "0.004": 1202, - "0.008": 1584, - "0.016": 1585, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 7921, + "1000": 41, + "128000": 7921, + "16000": 7921, + "2000": 751, + "256000": 7921, + "32000": 7921, + "4000": 3563, + "512000": 7921, + "64000": 7921, + "8000": 7836 }, - "count": 1585, - "sum": 5.924777266000009 + "count": 7921, + "sum": 32459685.32300009 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs", - "verb": "POST" + "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -5844,10 +5215,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "service", + "name": "serviceaccount_tokens_secret", "workqueue": { "adds": { - "count": 0 + "count": 45 }, "depth": { "count": 0 @@ -5860,27 +5231,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -5902,28 +5252,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 0, - "0.004": 688, - "0.008": 1569, - "0.016": 1582, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 182, + "1000": 0, + "128000": 182, + "16000": 173, + "2000": 76, + "256000": 182, + "32000": 182, + "4000": 86, + "512000": 182, + "64000": 182, + "8000": 113 }, - "count": 1585, - "sum": 6.8966293520000095 + "count": 182, + "sum": 1193234.1 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", - "verb": "DELETE" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -5940,10 +5290,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "node_lifecycle_controller_pods", + "name": "pvprotection", "workqueue": { "adds": { - "count": 1600 + "count": 0 }, "depth": { "count": 0 @@ -5956,27 +5306,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1600, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 10, - "0.00009999999999999999": 1597, - "0.001": 1600, - "0.01": 1600, - "0.1": 1600, - "1": 1600, - "10": 1600 - }, - "count": 1600, - "sum": 0.03146652500000004 - } - } } } }, @@ -5997,29 +5326,12 @@ "MetricSetFields": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 44, - "0.002": 703, - "0.004": 3385, - "0.008": 7833, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.13671674199991 - } - } + "count": 67202 } }, - "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1?timeout=32s", - "verb": "GET" + "code": "200", + "host": "172.18.0.2:6443", + "method": "PUT" }, "Index": "", "ID": "", @@ -6039,28 +5351,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 33, - "0.004": 1552, - "0.008": 1585, - "0.016": 1585, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 1585, - "sum": 3.9986306189999987 + "count": 1, + "sum": 251192.335 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", - "verb": "DELETE" + "url": "https://172.18.0.2:6443/apis/apps/v1/replicasets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -6080,27 +5392,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.004304235 + "sum": 8323.121 } } } }, - "url": "https://172.18.0.2:6443/api/v1/limitranges?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/endpointslices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -6118,31 +5430,21 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.199746336 - } - } + "name": "claims", + "workqueue": { + "adds": { + "count": 6340 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/replicationcontrollers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -6162,27 +5464,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 1327, - "0.004": 1570, - "0.008": 1584, - "0.016": 1584, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 7921, + "1000": 18, + "128000": 7921, + "16000": 7921, + "2000": 561, + "256000": 7921, + "32000": 7921, + "4000": 3312, + "512000": 7921, + "64000": 7921, + "8000": 7839 }, - "count": 1585, - "sum": 2.8280449830000034 + "count": 7921, + "sum": 33337573.244000025 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -6196,30 +5498,6 @@ "Period": 0, "DisableTimeSeries": false }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "client": { - "request": { - "count": 67202 - } - }, - "code": "200", - "host": "172.18.0.2:6443", - "method": "PUT" - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, { "RootFields": null, "ModuleFields": null, @@ -6227,27 +5505,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 64, - "0.002": 876, - "0.004": 3720, - "0.008": 7848, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 32.02661821400003 + "count": 1, + "sum": 9554.270999999999 } } } }, - "url": "https://172.18.0.2:6443/api/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/policy/v1/poddisruptionbudgets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -6265,10 +5543,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "serviceaccount_tokens_service", + "name": "noexec_taint_node", "workqueue": { "adds": { - "count": 45 + "count": 1 }, "depth": { "count": 0 @@ -6276,32 +5554,8 @@ "longestrunning": { "sec": 0 }, - "retries": { - "count": 0 - }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 45, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 3, - "0.00009999999999999999": 32, - "0.001": 40, - "0.01": 45, - "0.1": 45, - "1": 45, - "10": 45 - }, - "count": 45, - "sum": 0.013587963 - } - } } } }, @@ -6320,45 +5574,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "disruption_recheck", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 72, + "128000": 7921, + "16000": 7921, + "2000": 998, + "256000": 7921, + "32000": 7921, + "4000": 3865, + "512000": 7921, + "64000": 7921, + "8000": 7858 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 31296825.181000117 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -6378,27 +5618,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 30, - "0.002": 628, - "0.004": 3430, - "0.008": 7839, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 33.331486811000026 + "count": 1, + "sum": 8447.322 } } } }, - "url": "https://172.18.0.2:6443/apis/policy/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/configmaps?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -6416,42 +5656,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "noexec_taint_pod", - "workqueue": { - "adds": { - "count": 4770 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 4770, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 2742, - "0.00009999999999999999": 4761, - "0.001": 4769, - "0.01": 4770, - "0.1": 4770, - "1": 4770, - "10": 4770 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 4770, - "sum": 0.057577061999999915 + "count": 1, + "sum": 6885.754 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/extensions/v1beta1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -6471,27 +5700,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 33, - "0.002": 503, - "0.004": 3059, - "0.008": 7828, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 7921, - "sum": 33.94266633799993 + "count": 1, + "sum": 298650.755 } } } }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/batch/v1/jobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -6512,27 +5741,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 31, - "0.002": 596, - "0.004": 3350, - "0.008": 7826, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 39, + "128000": 7921, + "16000": 7921, + "2000": 652, + "256000": 7921, + "32000": 7921, + "4000": 3406, + "512000": 7921, + "64000": 7921, + "8000": 7844 }, "count": 7921, - "sum": 33.25637538899997 + "sum": 32928653.598000012 } } } }, - "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -6550,42 +5779,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "node_lifecycle_controller", - "workqueue": { - "adds": { - "count": 317 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 317, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 279, - "0.00009999999999999999": 316, - "0.001": 317, - "0.01": 317, - "0.1": 317, - "1": 317, - "10": 317 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1581, + "2000": 0, + "256000": 1585, + "32000": 1583, + "4000": 712, + "512000": 1585, + "64000": 1584, + "8000": 1546 }, - "count": 317, - "sum": 0.0028504730000000005 + "count": 1585, + "sum": 6961083.008000002 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods", + "verb": "POST" }, "Index": "", "ID": "", @@ -6602,45 +5820,31 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "replicationmanager", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 555, + "256000": 7921, + "32000": 7921, + "4000": 3227, + "512000": 7921, + "64000": 7921, + "8000": 7830 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 33560942.23699999 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta2?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -6660,22 +5864,22 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.168521167 + "sum": 168521.167 } } } @@ -6698,10 +5902,34 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "endpoint_slice_mirroring", + "client": { + "request": { + "count": 1 + } + }, + "code": "403", + "host": "172.18.0.2:6443", + "method": "GET" + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "name": "serviceaccount_tokens_service", "workqueue": { "adds": { - "count": 6 + "count": 45 }, "depth": { "count": 0 @@ -6714,27 +5942,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 1, - "0.00009999999999999999": 6, - "0.001": 6, - "0.01": 6, - "0.1": 6, - "1": 6, - "10": 6 - }, - "count": 6, - "sum": 0.000104123 - } - } } } }, @@ -6756,27 +5963,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 24, - "0.002": 551, - "0.004": 3241, - "0.008": 7821, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 33.54918595299994 + "count": 1, + "sum": 10918.125 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apps/v1/daemonsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -6794,31 +6001,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 47243, - "0.001": 0, - "0.002": 433, - "0.004": 3292, - "0.008": 21617, - "0.016": 47143, - "0.032": 47174, - "0.064": 47240, - "0.128": 47241, - "0.256": 47241, - "0.512": 47241 - }, - "count": 47243, - "sum": 379.1705878869994 - } - } + "name": "resourcequota_primary", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -6835,31 +6035,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.005445045 - } - } + "name": "pvcprotection", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csinodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -6878,12 +6071,29 @@ "MetricSetFields": { "client": { "request": { - "count": 1 + "duration": { + "us": { + "bucket": { + "+Inf": 7921, + "1000": 26, + "128000": 7920, + "16000": 7920, + "2000": 532, + "256000": 7920, + "32000": 7920, + "4000": 3205, + "512000": 7921, + "64000": 7920, + "8000": 7835 + }, + "count": 7921, + "sum": 33847457.39399993 + } + } } }, - "code": "403", - "host": "172.18.0.2:6443", - "method": "GET" + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1beta1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -6903,27 +6113,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 182, - "0.001": 0, - "0.002": 76, - "0.004": 86, - "0.008": 113, - "0.016": 173, - "0.032": 182, - "0.064": 182, - "0.128": 182, - "0.256": 182, - "0.512": 182 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 182, - "sum": 1.1932341000000002 + "count": 1, + "sum": 9219.587 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/networkpolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" }, "Index": "", @@ -6944,28 +6154,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3437, - "0.001": 0, - "0.002": 183, - "0.004": 3001, - "0.008": 3270, - "0.016": 3435, - "0.032": 3435, - "0.064": 3437, - "0.128": 3437, - "0.256": 3437, - "0.512": 3437 + "+Inf": 7921, + "1000": 44, + "128000": 7920, + "16000": 7920, + "2000": 703, + "256000": 7921, + "32000": 7920, + "4000": 3385, + "512000": 7921, + "64000": 7920, + "8000": 7833 }, - "count": 3437, - "sum": 11.231488972000028 + "count": 7921, + "sum": 33136716.741999913 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1?timeout=32s", + "verb": "GET" }, "Index": "", "ID": "", @@ -6985,28 +6195,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 0, - "0.004": 712, - "0.008": 1546, - "0.016": 1581, - "0.032": 1583, - "0.064": 1584, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 1585, - "sum": 6.961083008000002 + "count": 1, + "sum": 5865.891 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods", - "verb": "POST" + "url": "https://172.18.0.2:6443/api/v1/endpoints?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -7023,10 +6233,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "job", + "name": "resourcequota_priority", "workqueue": { "adds": { - "count": 9938 + "count": 0 }, "depth": { "count": 0 @@ -7035,31 +6245,10 @@ "sec": 0 }, "retries": { - "count": 12734 + "count": 0 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 9938, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 715, - "0.00009999999999999999": 6406, - "0.001": 6496, - "0.01": 9130, - "0.1": 9937, - "1": 9938, - "10": 9938 - }, - "count": 9938, - "sum": 29.61603745899999 - } - } } } }, @@ -7078,10 +6267,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "garbage_collector_attempt_to_delete", + "name": "bootstrap_signer_queue", "workqueue": { "adds": { - "count": 1592 + "count": 1 }, "depth": { "count": 0 @@ -7094,27 +6283,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1592, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 50, - "0.1": 1592, - "1": 1592, - "10": 1592 - }, - "count": 1592, - "sum": 20.110782428000025 - } - } } } }, @@ -7135,29 +6303,12 @@ "MetricSetFields": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 2, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 - }, - "count": 3, - "sum": 0.020303007 - } - } + "count": 2 } }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/namespaces/%7Bnamespace%7D/endpointslices/%7Bname%7D", - "verb": "PUT" + "code": "404", + "host": "172.18.0.2:6443", + "method": "PATCH" }, "Index": "", "ID": "", @@ -7174,31 +6325,21 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 0, - "0.004": 3, - "0.008": 3, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 - }, - "count": 3, - "sum": 0.009478231 - } - } + "name": "node_lifecycle_controller", + "workqueue": { + "adds": { + "count": 317 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -7218,28 +6359,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 22, - "0.002": 552, - "0.004": 3260, - "0.008": 7839, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1585, + "2000": 0, + "256000": 1585, + "32000": 1585, + "4000": 1202, + "512000": 1585, + "64000": 1585, + "8000": 1584 }, - "count": 7921, - "sum": 33.45908608399998 + "count": 1585, + "sum": 5924777.266000009 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs", + "verb": "POST" }, "Index": "", "ID": "", @@ -7258,12 +6399,29 @@ "MetricSetFields": { "client": { "request": { - "count": 2 + "duration": { + "us": { + "bucket": { + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 + }, + "count": 1, + "sum": 51489.332 + } + } } }, - "code": "404", - "host": "172.18.0.2:6443", - "method": "PATCH" + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/leases?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" }, "Index": "", "ID": "", @@ -7280,10 +6438,10 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "ttlcontroller", + "name": "orphaned_pods_nodes", "workqueue": { "adds": { - "count": 317 + "count": 0 }, "depth": { "count": 0 @@ -7296,27 +6454,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 317, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 285, - "0.00009999999999999999": 317, - "0.001": 317, - "0.01": 317, - "0.1": 317, - "1": 317, - "10": 317 - }, - "count": 317, - "sum": 0.0017760000000000005 - } - } } } }, @@ -7335,72 +6472,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 23, - "0.002": 555, - "0.004": 3227, - "0.008": 7830, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.56094223699999 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta2?timeout=32s", - "verb": "GET" - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.005315358 - } - } + "name": "volume_expand", + "workqueue": { + "adds": { + "count": 1 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v1/horizontalpodautoscalers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -7417,7 +6506,7 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "name": "garbage_collector_attempt_to_orphan", + "name": "token_cleaner", "workqueue": { "adds": { "count": 0 @@ -7433,27 +6522,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } }, @@ -7475,68 +6543,27 @@ "client": { "request": { "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 - }, - "count": 1, - "sum": 0.684430812 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterrolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0, - "Period": 0, - "DisableTimeSeries": false - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 72, - "0.002": 998, - "0.004": 3865, - "0.008": 7858, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 59, + "128000": 7921, + "16000": 7921, + "2000": 864, + "256000": 7921, + "32000": 7921, + "4000": 3725, + "512000": 7921, + "64000": 7921, + "8000": 7849 }, "count": 7921, - "sum": 31.296825181000116 + "sum": 31881754.901999995 } } } }, - "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1beta1?timeout=32s", "verb": "GET" }, "Index": "", @@ -7554,31 +6581,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 6878, - "0.002": 7793, - "0.004": 7886, - "0.008": 7915, - "0.016": 7915, - "0.032": 7915, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 6.718848776000002 - } - } + "name": "node_lifecycle_controller_pods", + "workqueue": { + "adds": { + "count": 1600 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis?timeout=32s", - "verb": "GET" + } }, "Index": "", "ID": "", @@ -7595,31 +6615,24 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.251192335 - } - } + "name": "service", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/apps/v1/replicasets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } }, "Index": "", "ID": "", diff --git a/metricbeat/module/kubernetes/controllermanager/_meta/testdata/docs.plain-expected.json b/metricbeat/module/kubernetes/controllermanager/_meta/testdata/docs.plain-expected.json index c8f8a4801fd..35840548af8 100644 --- a/metricbeat/module/kubernetes/controllermanager/_meta/testdata/docs.plain-expected.json +++ b/metricbeat/module/kubernetes/controllermanager/_meta/testdata/docs.plain-expected.json @@ -1,4 +1,41 @@ [ + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "serviceaccount", + "workqueue": { + "adds": { + "count": 5 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, { "event": { "dataset": "kubernetes.controllermanager", @@ -10,27 +47,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.008447322 + "sum": 5445.045 } } } }, - "url": "https://172.18.0.2:6443/api/v1/configmaps?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csinodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -54,27 +91,71 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.584920724 + "sum": 9125.225 } } } }, - "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1/controllerrevisions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "client": { + "request": { + "duration": { + "us": { + "bucket": { + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 + }, + "count": 1, + "sum": 1480.236 + } + } + } + }, + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -98,27 +179,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 39, - "0.002": 732, - "0.004": 3565, - "0.008": 7856, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 29, + "128000": 7921, + "16000": 7921, + "2000": 575, + "256000": 7921, + "32000": 7921, + "4000": 3324, + "512000": 7921, + "64000": 7921, + "8000": 7833 }, "count": 7921, - "sum": 32.403059004999946 + "sum": 33364429.504999924 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -139,45 +220,31 @@ }, "kubernetes": { "controllermanager": { - "name": "certificate", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 1, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 862.9010000000001 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1/customresourcedefinitions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -200,27 +267,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 33, - "0.002": 503, - "0.004": 3059, - "0.008": 7828, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 33.94266633799993 + "count": 1, + "sum": 8447.322 } } } }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/configmaps?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -241,7 +308,7 @@ }, "kubernetes": { "controllermanager": { - "name": "disruption", + "name": "resourcequota_primary", "workqueue": { "adds": { "count": 0 @@ -257,29 +324,52 @@ }, "unfinished": { "sec": 0 - }, - "work": { + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 56, + "128000": 7921, + "16000": 7921, + "2000": 755, + "256000": 7921, + "32000": 7921, + "4000": 3433, + "512000": 7921, + "64000": 7921, + "8000": 7830 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 32762698.44599994 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -302,27 +392,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 23, - "0.002": 517, - "0.004": 3164, - "0.008": 7818, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.747268298 + "count": 1, + "sum": 5126.341 } } } }, - "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/namespaces?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -346,27 +436,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 19, - "0.002": 510, - "0.004": 3132, - "0.008": 7833, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 33.78700202399993 + "count": 1, + "sum": 10220.903 } } } }, - "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -390,27 +480,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.006118409 + "sum": 298650.755 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/deployments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/batch/v1/jobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -431,45 +521,31 @@ }, "kubernetes": { "controllermanager": { - "name": "resource_quota_controller_resource_changes", - "workqueue": { - "adds": { - "count": 13983 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 13983, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 82, - "0.000009999999999999999": 10299, - "0.00009999999999999999": 13968, - "0.001": 13983, - "0.01": 13983, - "0.1": 13983, - "1": 13983, - "10": 13983 + "+Inf": 7921, + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 555, + "256000": 7921, + "32000": 7921, + "4000": 3227, + "512000": 7921, + "64000": 7921, + "8000": 7830 }, - "count": 13983, - "sum": 0.10729375299999992 + "count": 7921, + "sum": 33560942.23699999 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta2?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -492,27 +568,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 30, + "128000": 7920, + "16000": 7920, + "2000": 512, + "256000": 7921, + "32000": 7920, + "4000": 3208, + "512000": 7921, + "64000": 7920, + "8000": 7823 }, - "count": 1, - "sum": 0.009219587 + "count": 7921, + "sum": 33784837.885 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/networkpolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -533,42 +609,31 @@ }, "kubernetes": { "controllermanager": { - "name": "noexec_taint_pod", - "workqueue": { - "adds": { - "count": 4770 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 4770, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 2742, - "0.00009999999999999999": 4761, - "0.001": 4769, - "0.01": 4770, - "0.1": 4770, - "1": 4770, - "10": 4770 + "+Inf": 7921, + "1000": 54, + "128000": 7920, + "16000": 7920, + "2000": 733, + "256000": 7920, + "32000": 7920, + "4000": 3497, + "512000": 7921, + "64000": 7920, + "8000": 7848 }, - "count": 4770, - "sum": 0.057577061999999915 + "count": 7921, + "sum": 33072480.697999936 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/policy/v1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -591,27 +656,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.008323121 + "sum": 5955 } } } }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/endpointslices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/batch/v1/cronjobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -634,12 +699,29 @@ "controllermanager": { "client": { "request": { - "count": 1 + "duration": { + "us": { + "bucket": { + "+Inf": 7921, + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 517, + "256000": 7921, + "32000": 7921, + "4000": 3164, + "512000": 7921, + "64000": 7921, + "8000": 7818 + }, + "count": 7921, + "sum": 33747268.298 + } + } } }, - "code": "403", - "host": "172.18.0.2:6443", - "method": "GET" + "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -662,27 +744,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 0, - "0.004": 3, - "0.008": 3, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 3, - "sum": 0.009478231 + "count": 1, + "sum": 7787.689 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1/events?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -703,10 +785,10 @@ }, "kubernetes": { "controllermanager": { - "name": "serviceaccount_tokens_service", + "name": "ClusterRoleAggregator", "workqueue": { "adds": { - "count": 45 + "count": 27 }, "depth": { "count": 0 @@ -719,27 +801,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 45, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 3, - "0.00009999999999999999": 32, - "0.001": 40, - "0.01": 45, - "0.1": 45, - "1": 45, - "10": 45 - }, - "count": 45, - "sum": 0.013587963 - } - } } } } @@ -763,70 +824,29 @@ "controllermanager": { "client": { "request": { - "count": 2 + "duration": { + "us": { + "bucket": { + "+Inf": 7921, + "1000": 30, + "128000": 7920, + "16000": 7920, + "2000": 628, + "256000": 7920, + "32000": 7920, + "4000": 3430, + "512000": 7921, + "64000": 7920, + "8000": 7839 + }, + "count": 7921, + "sum": 33331486.811000027 + } + } } }, - "code": "404", - "host": "172.18.0.2:6443", - "method": "PATCH" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "name": "pvcs", - "workqueue": { - "adds": { - "count": 2 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 2, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 2, - "0.001": 2, - "0.01": 2, - "0.1": 2, - "1": 2, - "10": 2 - }, - "count": 2, - "sum": 0.000036973 - } - } - } - } + "url": "https://172.18.0.2:6443/apis/policy/v1beta1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -849,27 +869,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 24, + "128000": 7921, + "16000": 7921, + "2000": 574, + "256000": 7921, + "32000": 7921, + "4000": 3334, + "512000": 7921, + "64000": 7921, + "8000": 7842 }, - "count": 1, - "sum": 0.004378482 + "count": 7921, + "sum": 33281374.038000047 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csidrivers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/autoscaling/v1?timeout=32s", "verb": "GET" } }, @@ -893,27 +913,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 72, - "0.002": 998, - "0.004": 3865, - "0.008": 7858, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 6878, + "128000": 7921, + "16000": 7915, + "2000": 7793, + "256000": 7921, + "32000": 7915, + "4000": 7886, + "512000": 7921, + "64000": 7921, + "8000": 7915 }, "count": 7921, - "sum": 31.296825181000116 + "sum": 6718848.776000002 } } } }, - "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis?timeout=32s", "verb": "GET" } }, @@ -934,31 +954,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 18, - "0.002": 524, - "0.004": 3158, - "0.008": 7831, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.75089559099998 - } - } + "name": "volume_expand", + "workqueue": { + "adds": { + "count": 1 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1?timeout=32s", - "verb": "GET" + } } }, "metricset": { @@ -978,10 +991,10 @@ }, "kubernetes": { "controllermanager": { - "name": "statefulset", + "name": "ttl_jobs_to_delete", "workqueue": { "adds": { - "count": 2 + "count": 0 }, "depth": { "count": 0 @@ -994,27 +1007,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 2, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 1, - "0.01": 2, - "0.1": 2, - "1": 2, - "10": 2 - }, - "count": 2, - "sum": 0.002338638 - } - } } } } @@ -1039,27 +1031,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 34, - "0.002": 647, - "0.004": 3450, - "0.008": 7840, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 33, + "128000": 7921, + "16000": 7921, + "2000": 503, + "256000": 7921, + "32000": 7921, + "4000": 3059, + "512000": 7921, + "64000": 7921, + "8000": 7828 }, "count": 7921, - "sum": 32.91060117799999 + "sum": 33942666.33799993 } } } }, - "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -1083,28 +1075,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 315, + "1000": 0, + "128000": 315, + "16000": 315, + "2000": 2, + "256000": 315, + "32000": 315, + "4000": 178, + "512000": 315, + "64000": 315, + "8000": 313 }, - "count": 1, - "sum": 0.010220903 + "count": 315, + "sum": 1259640.6289999997 } } } }, - "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1/certificatesigningrequests?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events/%7Bname%7D", + "verb": "PATCH" } }, "metricset": { @@ -1126,29 +1118,12 @@ "controllermanager": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 47237, - "0.001": 0, - "0.002": 1929, - "0.004": 29430, - "0.008": 46836, - "0.016": 47157, - "0.032": 47188, - "0.064": 47233, - "0.128": 47237, - "0.256": 47237, - "0.512": 47237 - }, - "count": 47237, - "sum": 182.52047819099928 - } - } + "count": 1 } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", - "verb": "PUT" + "code": "403", + "host": "172.18.0.2:6443", + "method": "GET" } }, "metricset": { @@ -1171,27 +1146,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 0, + "256000": 3, + "32000": 3, + "4000": 3, + "512000": 3, + "64000": 3, + "8000": 3 }, - "count": 1, - "sum": 0.005635227 + "count": 3, + "sum": 9478.231 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/statefulsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D", "verb": "GET" } }, @@ -1212,45 +1187,31 @@ }, "kubernetes": { "controllermanager": { - "name": "pvcprotection", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 9297.166000000001 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/persistentvolumes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -1273,27 +1234,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 39, + "128000": 7921, + "16000": 7921, + "2000": 652, + "256000": 7921, + "32000": 7921, + "4000": 3406, + "512000": 7921, + "64000": 7921, + "8000": 7844 }, - "count": 1, - "sum": 0.005445045 + "count": 7921, + "sum": 32928653.598000012 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csinodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -1317,27 +1278,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.004304235 + "sum": 9219.587 } } } }, - "url": "https://172.18.0.2:6443/api/v1/limitranges?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/networkpolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -1360,29 +1321,12 @@ "controllermanager": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 23, - "0.002": 556, - "0.004": 3303, - "0.008": 7838, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.36970588799976 - } - } + "count": 2 } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1?timeout=32s", - "verb": "GET" + "code": "404", + "host": "172.18.0.2:6443", + "method": "PATCH" } }, "metricset": { @@ -1405,27 +1349,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 59, - "0.002": 864, - "0.004": 3725, - "0.008": 7849, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 31.881754901999994 + "count": 1, + "sum": 4304.235 } } } }, - "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/limitranges?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -1449,27 +1393,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 1, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.000903169 + "sum": 482711.14999999997 } } } }, - "url": "https://172.18.0.2:6443/api/v1/podtemplates?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1/runtimeclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -1493,27 +1437,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 18, - "0.002": 561, - "0.004": 3312, - "0.008": 7839, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 182, + "1000": 0, + "128000": 182, + "16000": 173, + "2000": 76, + "256000": 182, + "32000": 182, + "4000": 86, + "512000": 182, + "64000": 182, + "8000": 113 }, - "count": 7921, - "sum": 33.337573244000026 + "count": 182, + "sum": 1193234.1 } } } }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D", "verb": "GET" } }, @@ -1537,27 +1481,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 23, - "0.002": 565, - "0.004": 3309, - "0.008": 7819, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.4067139199999 + "count": 1, + "sum": 1206.6209999999999 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/storageclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -1578,31 +1522,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.007787689 - } - } + "name": "orphaned_pods_nodes", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1/events?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -1625,27 +1562,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 47243, - "0.001": 0, - "0.002": 433, - "0.004": 3292, - "0.008": 21617, - "0.016": 47143, - "0.032": 47174, - "0.064": 47240, - "0.128": 47241, - "0.256": 47241, - "0.512": 47241 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 47243, - "sum": 379.1705878869994 + "count": 1, + "sum": 20868.19 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", + "url": "https://172.18.0.2:6443/api/v1/nodes", "verb": "GET" } }, @@ -1666,42 +1603,31 @@ }, "kubernetes": { "controllermanager": { - "name": "noexec_taint_node", - "workqueue": { - "adds": { - "count": 1 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 1, - "0.001": 1, - "0.01": 1, - "0.1": 1, - "1": 1, - "10": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.000021587 + "sum": 9554.270999999999 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/policy/v1/poddisruptionbudgets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -1724,28 +1650,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 17, - "0.002": 518, - "0.004": 3203, - "0.008": 7828, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 2, + "1000": 0, + "128000": 2, + "16000": 2, + "2000": 0, + "256000": 2, + "32000": 2, + "4000": 1, + "512000": 2, + "64000": 2, + "8000": 1 }, - "count": 7921, - "sum": 33.685464928000066 + "count": 2, + "sum": 16458.466 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1beta1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D/status", + "verb": "PUT" } }, "metricset": { @@ -1768,28 +1694,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 2, + "1000": 0, + "128000": 2, + "16000": 2, + "2000": 0, + "256000": 2, + "32000": 2, + "4000": 0, + "512000": 2, + "64000": 2, + "8000": 1 }, - "count": 1, - "sum": 0.001325518 + "count": 2, + "sum": 19134.250999999997 } } } }, - "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1/apiservices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1/tokenreviews", + "verb": "POST" } }, "metricset": { @@ -1812,27 +1738,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 21, + "128000": 7921, + "16000": 7921, + "2000": 536, + "256000": 7921, + "32000": 7921, + "4000": 3200, + "512000": 7921, + "64000": 7921, + "8000": 7830 }, - "count": 1, - "sum": 0.199746336 + "count": 7921, + "sum": 33739958.584 } } } }, - "url": "https://172.18.0.2:6443/api/v1/replicationcontrollers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -1853,45 +1779,31 @@ }, "kubernetes": { "controllermanager": { - "name": "endpoint_slice_mirroring", - "workqueue": { - "adds": { - "count": 6 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 6, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 1, - "0.00009999999999999999": 6, - "0.001": 6, - "0.01": 6, - "0.1": 6, - "1": 6, - "10": 6 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 6, - "sum": 0.000104123 + "count": 1, + "sum": 432324.775 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/mutatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -1911,45 +1823,31 @@ }, "kubernetes": { "controllermanager": { - "name": "ttlcontroller", - "workqueue": { - "adds": { - "count": 317 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 317, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 285, - "0.00009999999999999999": 317, - "0.001": 317, - "0.01": 317, - "0.1": 317, - "1": 317, - "10": 317 + "+Inf": 139, + "1000": 0, + "128000": 139, + "16000": 138, + "2000": 0, + "256000": 139, + "32000": 139, + "4000": 29, + "512000": 139, + "64000": 139, + "8000": 138 }, - "count": 317, - "sum": 0.0017760000000000005 + "count": 139, + "sum": 676824.807 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D/token", + "verb": "POST" } }, "metricset": { @@ -1969,10 +1867,10 @@ }, "kubernetes": { "controllermanager": { - "name": "DynamicCABundle-request-header", + "name": "endpoint_slice_mirroring", "workqueue": { "adds": { - "count": 1585 + "count": 6 }, "depth": { "count": 0 @@ -1985,27 +1883,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 1140, - "0.001": 1585, - "0.01": 1585, - "0.1": 1585, - "1": 1585, - "10": 1585 - }, - "count": 1585, - "sum": 0.128010803 - } - } } } } @@ -2030,28 +1907,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 0, - "0.004": 1202, - "0.008": 1584, - "0.016": 1585, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 1585, - "sum": 5.924777266000009 + "count": 1, + "sum": 383570.902 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs", - "verb": "POST" + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -2071,7 +1948,7 @@ }, "kubernetes": { "controllermanager": { - "name": "DynamicServingCertificateController", + "name": "DynamicCABundle-client-ca-bundle", "workqueue": { "adds": { "count": 1585 @@ -2087,27 +1964,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 4, - "0.00009999999999999999": 1565, - "0.001": 1585, - "0.01": 1585, - "0.1": 1585, - "1": 1585, - "10": 1585 - }, - "count": 1585, - "sum": 0.04380364099999996 - } - } } } } @@ -2132,27 +1988,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 565, + "256000": 7921, + "32000": 7921, + "4000": 3309, + "512000": 7921, + "64000": 7921, + "8000": 7819 }, - "count": 1, - "sum": 0.001286213 + "count": 7921, + "sum": 33406713.9199999 } } } }, - "url": "https://172.18.0.2:6443/healthz?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -2173,31 +2029,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 315, - "0.001": 0, - "0.002": 2, - "0.004": 178, - "0.008": 313, - "0.016": 315, - "0.032": 315, - "0.064": 315, - "0.128": 315, - "0.256": 315, - "0.512": 315 - }, - "count": 315, - "sum": 1.2596406289999997 - } - } + "name": "replicationmanager", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events/%7Bname%7D", - "verb": "PATCH" + } } }, "metricset": { @@ -2217,7 +2066,7 @@ }, "kubernetes": { "controllermanager": { - "name": "replicationmanager", + "name": "horizontalpodautoscaler", "workqueue": { "adds": { "count": 0 @@ -2233,27 +2082,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } } @@ -2278,28 +2106,65 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3200, - "0.001": 0, - "0.002": 190, - "0.004": 2389, - "0.008": 3174, - "0.016": 3196, - "0.032": 3199, - "0.064": 3200, - "0.128": 3200, - "0.256": 3200, - "0.512": 3200 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1584, + "2000": 1327, + "256000": 1585, + "32000": 1585, + "4000": 1570, + "512000": 1585, + "64000": 1585, + "8000": 1584 }, - "count": 3200, - "sum": 11.213909066999973 + "count": 1585, + "sum": 2828044.9830000033 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events", - "verb": "POST" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", + "verb": "GET" + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "resourcequota_priority", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 + } + } } }, "metricset": { @@ -2322,27 +2187,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 182, - "0.001": 0, - "0.002": 76, - "0.004": 86, - "0.008": 113, - "0.016": 173, - "0.032": 182, - "0.064": 182, - "0.128": 182, - "0.256": 182, - "0.512": 182 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 182, - "sum": 1.1932341000000002 + "count": 1, + "sum": 5315.358 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/autoscaling/v1/horizontalpodautoscalers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -2366,27 +2231,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.02086819 + "sum": 6161.415 } } } }, - "url": "https://172.18.0.2:6443/api/v1/nodes", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/volumeattachments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -2407,10 +2272,10 @@ }, "kubernetes": { "controllermanager": { - "name": "ephemeral_volume", + "name": "replicaset", "workqueue": { "adds": { - "count": 0 + "count": 9 }, "depth": { "count": 0 @@ -2423,27 +2288,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } } @@ -2468,27 +2312,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.334037013 + "sum": 1286.213 } } } }, - "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1/priorityclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/healthz?timeout=32s", "verb": "GET" } }, @@ -2509,31 +2353,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.006161415 - } - } + "name": "ephemeral_volume", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/volumeattachments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -2553,31 +2390,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 21, - "0.002": 480, - "0.004": 3157, - "0.008": 7835, - "0.016": 7920, - "0.032": 7920, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.900589353999976 - } - } + "name": "cronjob", + "workqueue": { + "adds": { + "count": 8259 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 8259 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1?timeout=32s", - "verb": "GET" + } } }, "metricset": { @@ -2600,27 +2430,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.011306875 + "sum": 7561.396 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterroles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/rolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -2644,27 +2474,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3440, - "0.001": 3, - "0.002": 649, - "0.004": 3006, - "0.008": 3373, - "0.016": 3436, - "0.032": 3439, - "0.064": 3440, - "0.128": 3440, - "0.256": 3440, - "0.512": 3440 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 3440, - "sum": 10.386672363999987 + "count": 1, + "sum": 734162.309 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", + "url": "https://172.18.0.2:6443/apis/policy/v1beta1/podsecuritypolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -2685,31 +2515,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.009554271 - } - } + "name": "garbage_collector_graph_changes", + "workqueue": { + "adds": { + "count": 190259 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/policy/v1/poddisruptionbudgets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -2729,31 +2552,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.001257167 - } - } + "name": "DynamicCABundle-request-header", + "workqueue": { + "adds": { + "count": 1585 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/serviceaccounts?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -2776,27 +2592,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 54, - "0.002": 733, - "0.004": 3497, - "0.008": 7848, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 + "1000": 23, + "128000": 7921, + "16000": 7921, + "2000": 556, + "256000": 7921, + "32000": 7921, + "4000": 3303, + "512000": 7921, + "64000": 7921, + "8000": 7838 }, "count": 7921, - "sum": 33.072480697999936 + "sum": 33369705.887999758 } } } }, - "url": "https://172.18.0.2:6443/apis/policy/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -2817,10 +2633,10 @@ }, "kubernetes": { "controllermanager": { - "name": "cronjob", + "name": "resource_quota_controller_resource_changes", "workqueue": { "adds": { - "count": 8259 + "count": 13983 }, "depth": { "count": 0 @@ -2829,31 +2645,47 @@ "sec": 0 }, "retries": { - "count": 8259 + "count": 0 }, "unfinished": { "sec": 0 + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "daemonset", + "workqueue": { + "adds": { + "count": 6 }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 8259, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 5664, - "0.1": 8258, - "1": 8259, - "10": 8259 - }, - "count": 8259, - "sum": 82.251850376 - } - } + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } } } @@ -2878,27 +2710,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.006885754 + "sum": 334037.013 } } } }, - "url": "https://172.18.0.2:6443/apis/extensions/v1beta1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1/priorityclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -2921,12 +2753,29 @@ "controllermanager": { "client": { "request": { - "count": 3170 + "duration": { + "us": { + "bucket": { + "+Inf": 1, + "1000": 1, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 + }, + "count": 1, + "sum": 903.169 + } + } } }, - "code": "200", - "host": "172.18.0.2:6443", - "method": "DELETE" + "url": "https://172.18.0.2:6443/api/v1/podtemplates?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -2949,27 +2798,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 6878, - "0.002": 7793, - "0.004": 7886, - "0.008": 7915, - "0.016": 7915, - "0.032": 7915, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 6.718848776000002 + "count": 1, + "sum": 1330.3519999999999 } } } }, - "url": "https://172.18.0.2:6443/apis?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/resourcequotas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -2993,27 +2842,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.001479616 + "sum": 11306.875 } } } }, - "url": "https://172.18.0.2:6443/api/v1/nodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterroles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -3037,27 +2886,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 44, - "0.002": 703, - "0.004": 3385, - "0.008": 7833, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7921, - "0.512": 7921 + "1000": 18, + "128000": 7921, + "16000": 7921, + "2000": 524, + "256000": 7921, + "32000": 7921, + "4000": 3158, + "512000": 7921, + "64000": 7921, + "8000": 7831 }, "count": 7921, - "sum": 33.13671674199991 + "sum": 33750895.59099998 } } } }, - "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -3081,27 +2930,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 39, - "0.002": 652, - "0.004": 3406, - "0.008": 7844, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 19, + "128000": 7921, + "16000": 7921, + "2000": 510, + "256000": 7921, + "32000": 7921, + "4000": 3132, + "512000": 7921, + "64000": 7921, + "8000": 7833 }, "count": 7921, - "sum": 32.92865359800001 + "sum": 33787002.02399993 } } } }, - "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -3114,6 +2963,43 @@ "type": "kubernetes" } }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "pvcs", + "workqueue": { + "adds": { + "count": 2 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, { "event": { "dataset": "kubernetes.controllermanager", @@ -3125,28 +3011,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 3200, + "1000": 0, + "128000": 3200, + "16000": 3196, + "2000": 190, + "256000": 3200, + "32000": 3199, + "4000": 2389, + "512000": 3200, + "64000": 3200, + "8000": 3174 }, - "count": 1, - "sum": 0.001480236 + "count": 3200, + "sum": 11213909.066999974 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/events", + "verb": "POST" } }, "metricset": { @@ -3169,27 +3055,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 26, - "0.002": 532, - "0.004": 3205, - "0.008": 7835, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.847457393999925 + "count": 1, + "sum": 6118.409 } } } }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apps/v1/deployments?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -3213,28 +3099,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 0, - "0.004": 712, - "0.008": 1546, - "0.016": 1581, - "0.032": 1583, - "0.064": 1584, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 47237, + "1000": 0, + "128000": 47237, + "16000": 47157, + "2000": 1929, + "256000": 47237, + "32000": 47188, + "4000": 29430, + "512000": 47237, + "64000": 47233, + "8000": 46836 }, - "count": 1585, - "sum": 6.961083008000002 + "count": 47237, + "sum": 182520478.1909993 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods", - "verb": "POST" + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", + "verb": "PUT" } }, "metricset": { @@ -3254,14 +3140,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "count": 6513 + "name": "pvprotection", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "code": "201", - "host": "172.18.0.2:6443", - "method": "POST" + } } }, "metricset": { @@ -3283,29 +3179,12 @@ "controllermanager": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 27, - "0.002": 621, - "0.004": 3345, - "0.008": 7858, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.03073436400006 - } - } + "count": 3170 } }, - "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1?timeout=32s", - "verb": "GET" + "code": "200", + "host": "172.18.0.2:6443", + "method": "DELETE" } }, "metricset": { @@ -3328,27 +3207,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.005913528 + "sum": 6885.754 } } } }, - "url": "https://172.18.0.2:6443/api/v1/services?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/extensions/v1beta1/ingresses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -3369,10 +3248,10 @@ }, "kubernetes": { "controllermanager": { - "name": "node_lifecycle_controller_pods", + "name": "namespace", "workqueue": { "adds": { - "count": 1600 + "count": 0 }, "depth": { "count": 0 @@ -3385,27 +3264,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1600, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 10, - "0.00009999999999999999": 1597, - "0.001": 1600, - "0.01": 1600, - "0.1": 1600, - "1": 1600, - "10": 1600 - }, - "count": 1600, - "sum": 0.03146652500000004 - } - } } } } @@ -3430,28 +3288,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 56, - "0.002": 755, - "0.004": 3433, - "0.008": 7830, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 0, + "256000": 3, + "32000": 3, + "4000": 0, + "512000": 3, + "64000": 3, + "8000": 2 }, - "count": 7921, - "sum": 32.76269844599994 + "count": 3, + "sum": 20303.007 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/namespaces/%7Bnamespace%7D/endpointslices/%7Bname%7D", + "verb": "PUT" } }, "metricset": { @@ -3471,45 +3329,31 @@ }, "kubernetes": { "controllermanager": { - "name": "garbage_collector_attempt_to_orphan", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1585, + "2000": 0, + "256000": 1585, + "32000": 1585, + "4000": 1202, + "512000": 1585, + "64000": 1585, + "8000": 1584 }, - "count": 0, - "sum": 0 + "count": 1585, + "sum": 5924777.266000009 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs", + "verb": "POST" } }, "metricset": { @@ -3531,29 +3375,12 @@ "controllermanager": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 40, - "0.002": 663, - "0.004": 3393, - "0.008": 7841, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 32.96648631899999 - } - } + "count": 6513 } }, - "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1?timeout=32s", - "verb": "GET" + "code": "201", + "host": "172.18.0.2:6443", + "method": "POST" } }, "metricset": { @@ -3573,10 +3400,44 @@ }, "kubernetes": { "controllermanager": { - "name": "garbage_collector_graph_changes", + "name": "noexec_taint_node", "workqueue": { "adds": { - "count": 190259 + "count": 1 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "unfinished": { + "sec": 0 + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "serviceaccount_tokens_service", + "workqueue": { + "adds": { + "count": 45 }, "depth": { "count": 0 @@ -3589,27 +3450,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 190259, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 312, - "0.000009999999999999999": 162187, - "0.00009999999999999999": 189930, - "0.001": 190257, - "0.01": 190259, - "0.1": 190259, - "1": 190259, - "10": 190259 - }, - "count": 190259, - "sum": 1.5498176499999963 - } - } } } } @@ -3634,27 +3474,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 24, - "0.002": 551, - "0.004": 3241, - "0.008": 7821, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 72, + "128000": 7921, + "16000": 7921, + "2000": 998, + "256000": 7921, + "32000": 7921, + "4000": 3865, + "512000": 7921, + "64000": 7921, + "8000": 7858 }, "count": 7921, - "sum": 33.54918595299994 + "sum": 31296825.181000117 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -3678,27 +3518,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.168521167 + "sum": 5865.891 } } } }, - "url": "https://172.18.0.2:6443/api/v1/persistentvolumeclaims?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/endpoints?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -3719,10 +3559,54 @@ }, "kubernetes": { "controllermanager": { - "name": "serviceaccount_tokens_secret", + "client": { + "request": { + "duration": { + "us": { + "bucket": { + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 3, + "256000": 3, + "32000": 3, + "4000": 3, + "512000": 3, + "64000": 3, + "8000": 3 + }, + "count": 3, + "sum": 5033.812000000001 + } + } + } + }, + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D", + "verb": "GET" + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "bootstrap_signer_queue", "workqueue": { "adds": { - "count": 45 + "count": 1 }, "depth": { "count": 0 @@ -3735,27 +3619,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 45, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 26, - "0.00009999999999999999": 29, - "0.001": 36, - "0.01": 45, - "0.1": 45, - "1": 45, - "10": 45 - }, - "count": 45, - "sum": 0.015241941000000002 - } - } } } } @@ -3777,31 +3640,21 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 - }, - "count": 1, - "sum": 0.48271115 - } - } + "name": "noexec_taint_pod", + "workqueue": { + "adds": { + "count": 4770 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/node.k8s.io/v1/runtimeclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -3824,28 +3677,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 0, - "0.004": 688, - "0.008": 1569, - "0.016": 1582, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 7921, + "1000": 26, + "128000": 7921, + "16000": 7921, + "2000": 576, + "256000": 7921, + "32000": 7921, + "4000": 3377, + "512000": 7921, + "64000": 7921, + "8000": 7840 }, - "count": 1585, - "sum": 6.8966293520000095 + "count": 7921, + "sum": 33167423.892999955 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", - "verb": "DELETE" + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1beta1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -3892,10 +3745,10 @@ }, "kubernetes": { "controllermanager": { - "name": "endpoint_slice", + "name": "serviceaccount_tokens_secret", "workqueue": { "adds": { - "count": 10 + "count": 45 }, "depth": { "count": 0 @@ -3904,31 +3757,10 @@ "sec": 0 }, "retries": { - "count": 7 + "count": 0 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 10, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 2, - "0.001": 7, - "0.01": 10, - "0.1": 10, - "1": 10, - "10": 10 - }, - "count": 10, - "sum": 0.022632885 - } - } } } } @@ -3953,27 +3785,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 26, - "0.002": 576, - "0.004": 3377, - "0.008": 7840, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 20, + "128000": 7921, + "16000": 7921, + "2000": 547, + "256000": 7921, + "32000": 7921, + "4000": 3182, + "512000": 7921, + "64000": 7921, + "8000": 7836 }, "count": 7921, - "sum": 33.167423892999956 + "sum": 33695198.23299993 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -3996,12 +3828,29 @@ "controllermanager": { "client": { "request": { - "count": 67202 + "duration": { + "us": { + "bucket": { + "+Inf": 47243, + "1000": 0, + "128000": 47241, + "16000": 47143, + "2000": 433, + "256000": 47241, + "32000": 47174, + "4000": 3292, + "512000": 47241, + "64000": 47240, + "8000": 21617 + }, + "count": 47243, + "sum": 379170587.88699937 + } + } } }, - "code": "200", - "host": "172.18.0.2:6443", - "method": "PUT" + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/namespaces/%7Bnamespace%7D/leases/%7Bname%7D?timeout=5s", + "verb": "GET" } }, "metricset": { @@ -4024,27 +3873,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 22, - "0.002": 552, - "0.004": 3260, - "0.008": 7839, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 41, + "128000": 7921, + "16000": 7921, + "2000": 751, + "256000": 7921, + "32000": 7921, + "4000": 3563, + "512000": 7921, + "64000": 7921, + "8000": 7836 }, "count": 7921, - "sum": 33.45908608399998 + "sum": 32459685.32300009 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -4065,45 +3914,58 @@ }, "kubernetes": { "controllermanager": { - "name": "volume_expand", - "workqueue": { - "adds": { - "count": 1 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { + "count": 67202 + } + }, + "code": "200", + "host": "172.18.0.2:6443", + "method": "PUT" + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 1, - "0.1": 1, - "1": 1, - "10": 1 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1585, + "2000": 33, + "256000": 1585, + "32000": 1585, + "4000": 1552, + "512000": 1585, + "64000": 1585, + "8000": 1585 }, - "count": 1, - "sum": 0.007196112 + "count": 1585, + "sum": 3998630.6189999986 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", + "verb": "DELETE" } }, "metricset": { @@ -4126,27 +3988,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "+Inf": 7921, + "1000": 27, + "128000": 7920, + "16000": 7920, + "2000": 569, + "256000": 7920, + "32000": 7920, + "4000": 3377, + "512000": 7921, + "64000": 7920, + "8000": 7832 }, - "count": 1, - "sum": 0.734162309 + "count": 7921, + "sum": 33491676.60099996 } } } }, - "url": "https://172.18.0.2:6443/apis/policy/v1beta1/podsecuritypolicies?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -4170,27 +4032,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 31, + "128000": 7921, + "16000": 7921, + "2000": 596, + "256000": 7921, + "32000": 7921, + "4000": 3350, + "512000": 7921, + "64000": 7921, + "8000": 7826 }, - "count": 1, - "sum": 0.005865891 + "count": 7921, + "sum": 33256375.38899997 } } } }, - "url": "https://172.18.0.2:6443/api/v1/endpoints?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -4211,10 +4073,10 @@ }, "kubernetes": { "controllermanager": { - "name": "service", + "name": "claims", "workqueue": { "adds": { - "count": 0 + "count": 6340 }, "depth": { "count": 0 @@ -4222,32 +4084,8 @@ "longestrunning": { "sec": 0 }, - "retries": { - "count": 0 - }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } } @@ -4261,6 +4099,29 @@ "type": "kubernetes" } }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "leader": { + "is_master": true + }, + "name": "kube-controller-manager" + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, { "event": { "dataset": "kubernetes.controllermanager", @@ -4272,27 +4133,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 126, - "0.002": 1070, - "0.004": 3721, - "0.008": 7839, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7920 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 32.028062373999965 + "count": 1, + "sum": 4378.482 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/csidrivers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -4313,7 +4174,7 @@ }, "kubernetes": { "controllermanager": { - "name": "token_cleaner", + "name": "pvcprotection", "workqueue": { "adds": { "count": 0 @@ -4329,29 +4190,52 @@ }, "unfinished": { "sec": 0 - }, - "work": { + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 251192.335 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/apps/v1/replicasets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -4371,45 +4255,31 @@ }, "kubernetes": { "controllermanager": { - "name": "horizontalpodautoscaler", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 2, + "256000": 3, + "32000": 3, + "4000": 3, + "512000": 3, + "64000": 3, + "8000": 3 }, - "count": 0, - "sum": 0 + "count": 3, + "sum": 4963.458 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D", + "verb": "GET" } }, "metricset": { @@ -4429,45 +4299,31 @@ }, "kubernetes": { "controllermanager": { - "name": "resourcequota_primary", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 577, + "128000": 7921, + "16000": 7921, + "2000": 2786, + "256000": 7921, + "32000": 7921, + "4000": 5511, + "512000": 7921, + "64000": 7921, + "8000": 7877 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 24079648.88199997 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -4490,28 +4346,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 2, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 3, - "sum": 0.020303007 + "count": 1, + "sum": 2144.357 } } } }, - "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/namespaces/%7Bnamespace%7D/endpointslices/%7Bname%7D", - "verb": "PUT" + "url": "https://172.18.0.2:6443/api/v1/secrets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -4534,28 +4390,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3437, - "0.001": 0, - "0.002": 183, - "0.004": 3001, - "0.008": 3270, - "0.016": 3435, - "0.032": 3435, - "0.064": 3437, - "0.128": 3437, - "0.256": 3437, - "0.512": 3437 + "+Inf": 7921, + "1000": 64, + "128000": 7921, + "16000": 7921, + "2000": 876, + "256000": 7921, + "32000": 7921, + "4000": 3720, + "512000": 7921, + "64000": 7921, + "8000": 7848 }, - "count": 3437, - "sum": 11.231488972000028 + "count": 7921, + "sum": 32026618.21400003 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/api/v1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -4578,28 +4434,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 33, - "0.004": 1552, - "0.008": 1585, - "0.016": 1585, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 + "+Inf": 3437, + "1000": 0, + "128000": 3437, + "16000": 3435, + "2000": 183, + "256000": 3437, + "32000": 3435, + "4000": 3001, + "512000": 3437, + "64000": 3437, + "8000": 3270 }, - "count": 1585, - "sum": 3.9986306189999987 + "count": 3437, + "sum": 11231488.972000027 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", - "verb": "DELETE" + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D/status", + "verb": "PUT" } }, "metricset": { @@ -4619,10 +4475,10 @@ }, "kubernetes": { "controllermanager": { - "name": "ClusterRoleAggregator", + "name": "disruption_recheck", "workqueue": { "adds": { - "count": 27 + "count": 0 }, "depth": { "count": 0 @@ -4635,27 +4491,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 27, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 5, - "0.001": 27, - "0.01": 27, - "0.1": 27, - "1": 27, - "10": 27 - }, - "count": 27, - "sum": 0.0066875810000000015 - } - } } } } @@ -4677,10 +4512,31 @@ }, "kubernetes": { "controllermanager": { - "leader": { - "is_master": true + "client": { + "request": { + "duration": { + "us": { + "bucket": { + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 + }, + "count": 1, + "sum": 8323.121 + } + } + } }, - "name": "kube-controller-manager" + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1/endpointslices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -4703,27 +4559,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 20, - "0.002": 547, - "0.004": 3182, - "0.008": 7836, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.69519823299993 + "count": 1, + "sum": 5913.528 } } } }, - "url": "https://172.18.0.2:6443/apis/certificates.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/api/v1/services?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -4744,31 +4600,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 18103, - "0.001": 0, - "0.002": 2459, - "0.004": 14393, - "0.008": 16165, - "0.016": 17812, - "0.032": 18088, - "0.064": 18102, - "0.128": 18103, - "0.256": 18103, - "0.512": 18103 - }, - "count": 18103, - "sum": 70.20654167600028 - } - } + "name": "node_lifecycle_controller_pods", + "workqueue": { + "adds": { + "count": 1600 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D/status", - "verb": "PUT" + } } }, "metricset": { @@ -4788,10 +4637,10 @@ }, "kubernetes": { "controllermanager": { - "name": "endpoint", + "name": "ttlcontroller", "workqueue": { "adds": { - "count": 7 + "count": 317 }, "depth": { "count": 0 @@ -4800,33 +4649,56 @@ "sec": 0 }, "retries": { - "count": 4 + "count": 0 }, "unfinished": { "sec": 0 - }, - "work": { + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 2, - "0.001": 4, - "0.01": 7, - "0.1": 7, - "1": 7, - "10": 7 + "+Inf": 3, + "1000": 0, + "128000": 3, + "16000": 3, + "2000": 0, + "256000": 3, + "32000": 3, + "4000": 0, + "512000": 3, + "64000": 3, + "8000": 2 }, - "count": 7, - "sum": 0.022435438000000002 + "count": 3, + "sum": 20853.000999999997 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/endpoints/%7Bname%7D", + "verb": "PUT" } }, "metricset": { @@ -4849,27 +4721,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.001206621 + "sum": 168521.167 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1/storageclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/persistentvolumeclaims?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -4893,27 +4765,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 39, + "128000": 7921, + "16000": 7921, + "2000": 732, + "256000": 7921, + "32000": 7921, + "4000": 3565, + "512000": 7921, + "64000": 7921, + "8000": 7856 }, - "count": 1, - "sum": 0.001330352 + "count": 7921, + "sum": 32403059.004999947 } } } }, - "url": "https://172.18.0.2:6443/api/v1/resourcequotas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1?timeout=32s", "verb": "GET" } }, @@ -4937,27 +4809,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 31, + "128000": 7921, + "16000": 7920, + "2000": 549, + "256000": 7921, + "32000": 7920, + "4000": 3288, + "512000": 7921, + "64000": 7920, + "8000": 7836 }, - "count": 1, - "sum": 0.005955 + "count": 7921, + "sum": 33411074.78999993 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/cronjobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -4981,27 +4853,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.007561396 + "sum": 1325.518 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/rolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1/apiservices?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -5022,10 +4894,54 @@ }, "kubernetes": { "controllermanager": { - "name": "claims", + "client": { + "request": { + "duration": { + "us": { + "bucket": { + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 + }, + "count": 1, + "sum": 584920.7239999999 + } + } + } + }, + "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/flowschemas?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "DynamicServingCertificateController", "workqueue": { "adds": { - "count": 6340 + "count": 1585 }, "depth": { "count": 0 @@ -5033,29 +4949,48 @@ "longestrunning": { "sec": 0 }, + "retries": { + "count": 0 + }, "unfinished": { "sec": 0 + } + } + } + }, + "metricset": { + "name": "controllermanager", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "type": "kubernetes" + } + }, + { + "event": { + "dataset": "kubernetes.controllermanager", + "duration": 115000, + "module": "kubernetes" + }, + "kubernetes": { + "controllermanager": { + "name": "service", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6340, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 3961, - "0.001": 6335, - "0.01": 6340, - "0.1": 6340, - "1": 6340, - "10": 6340 - }, - "count": 6340, - "sum": 0.6305873910000022 - } - } + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } } } @@ -5080,28 +5015,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 3, - "0.004": 3, - "0.008": 3, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 2, + "1000": 1, + "128000": 2, + "16000": 2, + "2000": 2, + "256000": 2, + "32000": 2, + "4000": 2, + "512000": 2, + "64000": 2, + "8000": 2 }, - "count": 3, - "sum": 0.005033812 + "count": 2, + "sum": 2408.319 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s", + "verb": "POST" } }, "metricset": { @@ -5121,31 +5056,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.101172363 - } - } + "name": "certificate", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/pods?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -5168,27 +5096,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 1361, - "0.002": 1675, - "0.004": 2198, - "0.008": 3799, - "0.016": 7486, - "0.032": 7907, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 59, + "128000": 7921, + "16000": 7921, + "2000": 864, + "256000": 7921, + "32000": 7921, + "4000": 3725, + "512000": 7921, + "64000": 7921, + "8000": 7849 }, "count": 7921, - "sum": 64.48029232100004 + "sum": 31881754.901999995 } } } }, - "url": "https://172.18.0.2:6443/api?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apiregistration.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -5212,27 +5140,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 3440, + "1000": 3, + "128000": 3440, + "16000": 3436, + "2000": 649, + "256000": 3440, + "32000": 3439, + "4000": 3006, + "512000": 3440, + "64000": 3440, + "8000": 3373 }, - "count": 1, - "sum": 0.010918125 + "count": 3440, + "sum": 10386672.363999987 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/daemonsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/jobs/%7Bname%7D", "verb": "GET" } }, @@ -5253,7 +5181,7 @@ }, "kubernetes": { "controllermanager": { - "name": "volumes", + "name": "DynamicCABundle-csr-controller", "workqueue": { "adds": { "count": 6340 @@ -5264,29 +5192,11 @@ "longestrunning": { "sec": 0 }, + "retries": { + "count": 0 + }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6340, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 5380, - "0.001": 6340, - "0.01": 6340, - "0.1": 6340, - "1": 6340, - "10": 6340 - }, - "count": 6340, - "sum": 0.4856128449999997 - } - } } } } @@ -5308,31 +5218,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.009297166 - } - } + "name": "garbage_collector_attempt_to_delete", + "workqueue": { + "adds": { + "count": 1592 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1/persistentvolumes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + } } }, "metricset": { @@ -5352,10 +5255,10 @@ }, "kubernetes": { "controllermanager": { - "name": "daemonset", + "name": "disruption", "workqueue": { "adds": { - "count": 6 + "count": 0 }, "depth": { "count": 0 @@ -5368,27 +5271,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 4, - "0.01": 6, - "0.1": 6, - "1": 6, - "10": 6 - }, - "count": 6, - "sum": 0.005056306000000001 - } - } } } } @@ -5413,27 +5295,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.632319982 + "sum": 5635.227 } } } }, - "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingressclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/apps/v1/statefulsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -5457,27 +5339,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 23, - "0.002": 555, - "0.004": 3227, - "0.008": 7830, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 44, + "128000": 7920, + "16000": 7920, + "2000": 703, + "256000": 7921, + "32000": 7920, + "4000": 3385, + "512000": 7921, + "64000": 7920, + "8000": 7833 }, "count": 7921, - "sum": 33.56094223699999 + "sum": 33136716.741999913 } } } }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta2?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -5501,27 +5383,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "+Inf": 7921, + "1000": 40, + "128000": 7921, + "16000": 7921, + "2000": 663, + "256000": 7921, + "32000": 7921, + "4000": 3393, + "512000": 7921, + "64000": 7921, + "8000": 7841 }, - "count": 1, - "sum": 0.432324775 + "count": 7921, + "sum": 32966486.31899999 } } } }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/mutatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -5545,27 +5427,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 2, - "0.004": 3, - "0.008": 3, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 3, - "sum": 0.004963458 + "count": 1, + "sum": 1257.167 } } } }, - "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D", + "url": "https://172.18.0.2:6443/api/v1/serviceaccounts?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -5589,28 +5471,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, - "count": 2, - "sum": 0.016458466 + "count": 1, + "sum": 101172.363 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/replicasets/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/api/v1/pods?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -5630,45 +5512,31 @@ }, "kubernetes": { "controllermanager": { - "name": "deployment", - "workqueue": { - "adds": { - "count": 205 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 198 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 205, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 167, - "0.01": 205, - "0.1": 205, - "1": 205, - "10": 205 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 205, - "sum": 0.152872125 + "count": 1, + "sum": 6075.867 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/roles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -5688,45 +5556,31 @@ }, "kubernetes": { "controllermanager": { - "name": "root_ca_cert_publisher", - "workqueue": { - "adds": { - "count": 10 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 10, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 2, - "0.00009999999999999999": 10, - "0.001": 10, - "0.01": 10, - "0.1": 10, - "1": 10, - "10": 10 + "+Inf": 7921, + "1000": 22, + "128000": 7921, + "16000": 7921, + "2000": 552, + "256000": 7921, + "32000": 7921, + "4000": 3260, + "512000": 7921, + "64000": 7921, + "8000": 7839 }, - "count": 10, - "sum": 0.00032599999999999996 + "count": 7921, + "sum": 33459086.083999977 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/batch/v1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -5749,27 +5603,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 28, - "0.002": 617, - "0.004": 3327, - "0.008": 7832, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, - "count": 7921, - "sum": 33.451801867 + "count": 1, + "sum": 10918.125 } } } }, - "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/apps/v1/daemonsets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -5792,29 +5646,12 @@ "controllermanager": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.005126341 - } - } + "count": 1585 } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "code": "409", + "host": "172.18.0.2:6443", + "method": "PUT" } }, "metricset": { @@ -5837,27 +5674,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 1361, + "128000": 7921, + "16000": 7486, + "2000": 1675, + "256000": 7921, + "32000": 7907, + "4000": 2198, + "512000": 7921, + "64000": 7921, + "8000": 3799 }, - "count": 1, - "sum": 0.251192335 + "count": 7921, + "sum": 64480292.32100003 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/replicasets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api?timeout=32s", "verb": "GET" } }, @@ -5881,27 +5718,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 0, + "8000": 0 }, "count": 1, - "sum": 0.051489332 + "sum": 199746.336 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/leases?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/api/v1/replicationcontrollers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -5922,10 +5759,10 @@ }, "kubernetes": { "controllermanager": { - "name": "job", + "name": "endpoint_slice", "workqueue": { "adds": { - "count": 9938 + "count": 10 }, "depth": { "count": 0 @@ -5934,31 +5771,10 @@ "sec": 0 }, "retries": { - "count": 12734 + "count": 7 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 9938, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 715, - "0.00009999999999999999": 6406, - "0.001": 6496, - "0.01": 9130, - "0.1": 9937, - "1": 9938, - "10": 9938 - }, - "count": 9938, - "sum": 29.61603745899999 - } - } } } } @@ -5983,28 +5799,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 + "+Inf": 7921, + "1000": 21, + "128000": 7921, + "16000": 7920, + "2000": 480, + "256000": 7921, + "32000": 7920, + "4000": 3157, + "512000": 7921, + "64000": 7921, + "8000": 7835 }, - "count": 2, - "sum": 0.012623510000000001 + "count": 7921, + "sum": 33900589.35399998 } } } }, - "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D/status", - "verb": "PUT" + "url": "https://172.18.0.2:6443/apis/discovery.k8s.io/v1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -6024,31 +5840,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 27, - "0.002": 569, - "0.004": 3377, - "0.008": 7832, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.49167660099996 - } - } + "name": "job", + "workqueue": { + "adds": { + "count": 9938 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 12734 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1?timeout=32s", - "verb": "GET" + } } }, "metricset": { @@ -6068,10 +5877,10 @@ }, "kubernetes": { "controllermanager": { - "name": "serviceaccount", + "name": "statefulset", "workqueue": { "adds": { - "count": 5 + "count": 2 }, "depth": { "count": 0 @@ -6084,27 +5893,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 5, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 4, - "0.00009999999999999999": 5, - "0.001": 5, - "0.01": 5, - "0.1": 5, - "1": 5, - "10": 5 - }, - "count": 5, - "sum": 0.000074208 - } - } } } } @@ -6129,27 +5917,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 52, + "128000": 7920, + "16000": 7920, + "2000": 739, + "256000": 7920, + "32000": 7920, + "4000": 3489, + "512000": 7921, + "64000": 7920, + "8000": 7832 }, - "count": 1, - "sum": 0.006075867 + "count": 7921, + "sum": 32978230.279000036 } } } }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/roles?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/extensions/v1beta1?timeout=32s", "verb": "GET" } }, @@ -6173,27 +5961,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 30, - "0.002": 628, - "0.004": 3430, - "0.008": 7839, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 7921, - "sum": 33.331486811000026 + "count": 1, + "sum": 632319.982 } } } }, - "url": "https://172.18.0.2:6443/apis/policy/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1/ingressclasses?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -6214,10 +6002,10 @@ }, "kubernetes": { "controllermanager": { - "name": "DynamicCABundle-client-ca-bundle", + "name": "garbage_collector_attempt_to_orphan", "workqueue": { "adds": { - "count": 1585 + "count": 0 }, "depth": { "count": 0 @@ -6230,27 +6018,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 1101, - "0.001": 1585, - "0.01": 1585, - "0.1": 1585, - "1": 1585, - "10": 1585 - }, - "count": 1585, - "sum": 0.1301518080000002 - } - } } } } @@ -6275,28 +6042,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 577, - "0.002": 2786, - "0.004": 5511, - "0.008": 7877, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 2, + "1000": 0, + "128000": 2, + "16000": 2, + "2000": 0, + "256000": 2, + "32000": 2, + "4000": 1, + "512000": 2, + "64000": 2, + "8000": 1 }, - "count": 7921, - "sum": 24.07964888199997 + "count": 2, + "sum": 12623.51 } } } }, - "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1?timeout=32s", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/apps/v1/namespaces/%7Bnamespace%7D/deployments/%7Bname%7D/status", + "verb": "PUT" } }, "metricset": { @@ -6318,12 +6085,29 @@ "controllermanager": { "client": { "request": { - "count": 1585 + "duration": { + "us": { + "bucket": { + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1582, + "2000": 0, + "256000": 1585, + "32000": 1585, + "4000": 688, + "512000": 1585, + "64000": 1585, + "8000": 1569 + }, + "count": 1585, + "sum": 6896629.352000009 + } + } } }, - "code": "409", - "host": "172.18.0.2:6443", - "method": "PUT" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", + "verb": "DELETE" } }, "metricset": { @@ -6346,28 +6130,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 2, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 + "+Inf": 7921, + "1000": 28, + "128000": 7920, + "16000": 7920, + "2000": 617, + "256000": 7921, + "32000": 7920, + "4000": 3327, + "512000": 7921, + "64000": 7920, + "8000": 7832 }, - "count": 2, - "sum": 0.019134250999999998 + "count": 7921, + "sum": 33451801.867 } } } }, - "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1/tokenreviews", - "verb": "POST" + "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1beta1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -6387,45 +6171,31 @@ }, "kubernetes": { "controllermanager": { - "name": "orphaned_pods_nodes", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 684430.812 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterrolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -6448,27 +6218,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 31, - "0.002": 549, - "0.004": 3288, - "0.008": 7836, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 24, + "128000": 7921, + "16000": 7921, + "2000": 551, + "256000": 7921, + "32000": 7921, + "4000": 3241, + "512000": 7921, + "64000": 7921, + "8000": 7821 }, "count": 7921, - "sum": 33.41107478999993 + "sum": 33549185.952999942 } } } }, - "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -6489,45 +6259,31 @@ }, "kubernetes": { "controllermanager": { - "name": "resourcequota_priority", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 6997.829 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/persistentvolumes/%7Bname%7D", + "verb": "GET" } }, "metricset": { @@ -6547,45 +6303,31 @@ }, "kubernetes": { "controllermanager": { - "name": "namespace", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 7921, + "1000": 26, + "128000": 7920, + "16000": 7920, + "2000": 532, + "256000": 7920, + "32000": 7920, + "4000": 3205, + "512000": 7921, + "64000": 7920, + "8000": 7835 }, - "count": 0, - "sum": 0 + "count": 7921, + "sum": 33847457.39399993 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1beta1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -6608,22 +6350,22 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, "count": 1, - "sum": 0.003386321 + "sum": 3386.321 } } } @@ -6652,27 +6394,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 7921, + "1000": 126, + "128000": 7920, + "16000": 7920, + "2000": 1070, + "256000": 7920, + "32000": 7920, + "4000": 3721, + "512000": 7920, + "64000": 7920, + "8000": 7839 }, - "count": 1, - "sum": 0.005936288 + "count": 7921, + "sum": 32028062.373999964 } } } }, - "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1/csistoragecapacities?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/networking.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -6696,28 +6438,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "+Inf": 18103, + "1000": 0, + "128000": 18103, + "16000": 17812, + "2000": 2459, + "256000": 18103, + "32000": 18088, + "4000": 14393, + "512000": 18103, + "64000": 18102, + "8000": 16165 }, - "count": 1, - "sum": 0.006997829 + "count": 18103, + "sum": 70206541.67600028 } } } }, - "url": "https://172.18.0.2:6443/api/v1/persistentvolumes/%7Bname%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/apis/batch/v1/namespaces/%7Bnamespace%7D/cronjobs/%7Bname%7D/status", + "verb": "PUT" } }, "metricset": { @@ -6740,27 +6482,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 + "1000": 0, + "128000": 1, + "16000": 0, + "2000": 0, + "256000": 1, + "32000": 0, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 0 }, "count": 1, - "sum": 0.005315358 + "sum": 51489.332 } } } }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v1/horizontalpodautoscalers?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1/leases?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -6781,10 +6523,10 @@ }, "kubernetes": { "controllermanager": { - "name": "replicaset", + "name": "deployment", "workqueue": { "adds": { - "count": 9 + "count": 205 }, "depth": { "count": 0 @@ -6793,31 +6535,10 @@ "sec": 0 }, "retries": { - "count": 0 + "count": 198 }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 9, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 3, - "0.001": 7, - "0.01": 8, - "0.1": 9, - "1": 9, - "10": 9 - }, - "count": 9, - "sum": 0.020865991 - } - } } } } @@ -6842,27 +6563,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 + "+Inf": 7921, + "1000": 17, + "128000": 7921, + "16000": 7921, + "2000": 518, + "256000": 7921, + "32000": 7921, + "4000": 3203, + "512000": 7921, + "64000": 7921, + "8000": 7828 }, - "count": 1, - "sum": 0.533260944 + "count": 7921, + "sum": 33685464.92800006 } } } }, - "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/prioritylevelconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/batch/v1beta1?timeout=32s", "verb": "GET" } }, @@ -6883,31 +6604,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 30, - "0.002": 512, - "0.004": 3208, - "0.008": 7823, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.784837884999995 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1?timeout=32s", - "verb": "GET" + "name": "endpoint", + "workqueue": { + "adds": { + "count": 7 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 4 + }, + "unfinished": { + "sec": 0 + } + } } }, "metricset": { @@ -6930,71 +6644,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 29, - "0.002": 575, - "0.004": 3324, - "0.008": 7833, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 34, + "128000": 7921, + "16000": 7921, + "2000": 647, + "256000": 7921, + "32000": 7921, + "4000": 3450, + "512000": 7921, + "64000": 7921, + "8000": 7840 }, "count": 7921, - "sum": 33.364429504999926 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/scheduling.k8s.io/v1beta1?timeout=32s", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 1, - "0.002": 1, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.000862901 + "sum": 32910601.177999992 } } } }, - "url": "https://172.18.0.2:6443/apis/apiextensions.k8s.io/v1/customresourcedefinitions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1beta1?timeout=32s", "verb": "GET" } }, @@ -7015,10 +6685,10 @@ }, "kubernetes": { "controllermanager": { - "name": "pvprotection", + "name": "root_ca_cert_publisher", "workqueue": { "adds": { - "count": 0 + "count": 10 }, "depth": { "count": 0 @@ -7031,27 +6701,6 @@ }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } } @@ -7073,42 +6722,31 @@ }, "kubernetes": { "controllermanager": { - "name": "node_lifecycle_controller", - "workqueue": { - "adds": { - "count": 317 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 317, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 279, - "0.00009999999999999999": 316, - "0.001": 317, - "0.01": 317, - "0.1": 317, - "1": 317, - "10": 317 + "+Inf": 7921, + "1000": 18, + "128000": 7921, + "16000": 7921, + "2000": 561, + "256000": 7921, + "32000": 7921, + "4000": 3312, + "512000": 7921, + "64000": 7921, + "8000": 7839 }, - "count": 317, - "sum": 0.0028504730000000005 + "count": 7921, + "sum": 33337573.244000025 } } } - } + }, + "url": "https://172.18.0.2:6443/apis/autoscaling/v2beta1?timeout=32s", + "verb": "GET" } }, "metricset": { @@ -7128,45 +6766,31 @@ }, "kubernetes": { "controllermanager": { - "name": "ttl_jobs_to_delete", - "workqueue": { - "adds": { - "count": 0 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { + "client": { + "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 1, + "256000": 1, + "32000": 1, + "4000": 1, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 0, - "sum": 0 + "count": 1, + "sum": 1479.616 } } } - } + }, + "url": "https://172.18.0.2:6443/api/v1/nodes?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -7189,27 +6813,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 7921, - "0.001": 21, - "0.002": 536, - "0.004": 3200, - "0.008": 7830, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "+Inf": 1, + "1000": 0, + "128000": 1, + "16000": 1, + "2000": 0, + "256000": 1, + "32000": 1, + "4000": 0, + "512000": 1, + "64000": 1, + "8000": 1 }, - "count": 7921, - "sum": 33.739958584 + "count": 1, + "sum": 5936.288 } } } }, - "url": "https://172.18.0.2:6443/apis/coordination.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/storage.k8s.io/v1beta1/csistoragecapacities?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", "verb": "GET" } }, @@ -7230,31 +6854,44 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 1, - "0.008": 1, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.002144357 - } + "node": { + "collector": { + "count": 1, + "eviction": { + "count": 0 + }, + "health": { + "pct": 100 + }, + "unhealthy": { + "count": 0 } } }, - "url": "https://172.18.0.2:6443/api/v1/secrets?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "process": { + "cpu": { + "sec": 2461 + }, + "fds": { + "max": { + "count": 1048576 + }, + "open": { + "count": 17 + } + }, + "memory": { + "resident": { + "bytes": 108044288 + }, + "virtual": { + "bytes": 788058112 + } + }, + "started": { + "sec": 1655707927.95 + } + } } }, "metricset": { @@ -7274,31 +6911,24 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 64, - "0.002": 876, - "0.004": 3720, - "0.008": 7848, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 32.02661821400003 - } - } + "name": "token_cleaner", + "workqueue": { + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + }, + "unfinished": { + "sec": 0 } - }, - "url": "https://172.18.0.2:6443/api/v1?timeout=32s", - "verb": "GET" + } } }, "metricset": { @@ -7321,28 +6951,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 139, - "0.001": 0, - "0.002": 0, - "0.004": 29, - "0.008": 138, - "0.016": 138, - "0.032": 139, - "0.064": 139, - "0.128": 139, - "0.256": 139, - "0.512": 139 + "+Inf": 1, + "1000": 0, + "128000": 0, + "16000": 0, + "2000": 0, + "256000": 0, + "32000": 0, + "4000": 0, + "512000": 0, + "64000": 0, + "8000": 0 }, - "count": 139, - "sum": 0.676824807 + "count": 1, + "sum": 533260.944 } } } }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/serviceaccounts/%7Bname%7D/token", - "verb": "POST" + "url": "https://172.18.0.2:6443/apis/flowcontrol.apiserver.k8s.io/v1beta1/prioritylevelconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", + "verb": "GET" } }, "metricset": { @@ -7364,29 +6994,12 @@ "controllermanager": { "client": { "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 2, - "0.001": 1, - "0.002": 2, - "0.004": 2, - "0.008": 2, - "0.016": 2, - "0.032": 2, - "0.064": 2, - "0.128": 2, - "0.256": 2, - "0.512": 2 - }, - "count": 2, - "sum": 0.002408319 - } - } + "count": 313 } }, - "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1/subjectaccessreviews?timeout=10s", - "verb": "POST" + "code": "200", + "host": "172.18.0.2:6443", + "method": "PATCH" } }, "metricset": { @@ -7409,28 +7022,28 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 + "+Inf": 1585, + "1000": 0, + "128000": 1585, + "16000": 1581, + "2000": 0, + "256000": 1585, + "32000": 1583, + "4000": 712, + "512000": 1585, + "64000": 1584, + "8000": 1546 }, - "count": 1, - "sum": 0.383570902 + "count": 1585, + "sum": 6961083.008000002 } } } }, - "url": "https://172.18.0.2:6443/apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" + "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods", + "verb": "POST" } }, "metricset": { @@ -7450,10 +7063,10 @@ }, "kubernetes": { "controllermanager": { - "name": "disruption_recheck", + "name": "volumes", "workqueue": { "adds": { - "count": 0 + "count": 6340 }, "depth": { "count": 0 @@ -7461,32 +7074,8 @@ "longestrunning": { "sec": 0 }, - "retries": { - "count": 0 - }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 0, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 0, - "0.1": 0, - "1": 0, - "10": 0 - }, - "count": 0, - "sum": 0 - } - } } } } @@ -7511,27 +7100,27 @@ "client": { "request": { "duration": { - "sec": { + "us": { "bucket": { "+Inf": 7921, - "0.001": 31, - "0.002": 596, - "0.004": 3350, - "0.008": 7826, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 + "1000": 27, + "128000": 7921, + "16000": 7921, + "2000": 621, + "256000": 7921, + "32000": 7921, + "4000": 3345, + "512000": 7921, + "64000": 7921, + "8000": 7858 }, "count": 7921, - "sum": 33.25637538899997 + "sum": 33030734.36400006 } } } }, - "url": "https://172.18.0.2:6443/apis/authorization.k8s.io/v1beta1?timeout=32s", + "url": "https://172.18.0.2:6443/apis/authentication.k8s.io/v1?timeout=32s", "verb": "GET" } }, @@ -7552,562 +7141,10 @@ }, "kubernetes": { "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 41, - "0.002": 751, - "0.004": 3563, - "0.008": 7836, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 32.45968532300009 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/events.k8s.io/v1?timeout=32s", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 3, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 2, - "0.016": 3, - "0.032": 3, - "0.064": 3, - "0.128": 3, - "0.256": 3, - "0.512": 3 - }, - "count": 3, - "sum": 0.020853000999999996 - } - } - } - }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/endpoints/%7Bname%7D", - "verb": "PUT" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "name": "DynamicCABundle-csr-controller", - "workqueue": { - "adds": { - "count": 6340 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 6340, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 20, - "0.001": 6320, - "0.01": 6340, - "0.1": 6340, - "1": 6340, - "10": 6340 - }, - "count": 6340, - "sum": 2.040141396000006 - } - } - } - } - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1585, - "0.001": 0, - "0.002": 1327, - "0.004": 1570, - "0.008": 1584, - "0.016": 1584, - "0.032": 1585, - "0.064": 1585, - "0.128": 1585, - "0.256": 1585, - "0.512": 1585 - }, - "count": 1585, - "sum": 2.8280449830000034 - } - } - } - }, - "url": "https://172.18.0.2:6443/api/v1/namespaces/%7Bnamespace%7D/pods/%7Bname%7D", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "name": "bootstrap_signer_queue", - "workqueue": { - "adds": { - "count": 1 - }, - "depth": { - "count": 0 - }, - "longestrunning": { - "sec": 0 - }, - "retries": { - "count": 0 - }, - "unfinished": { - "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 1, - "0.01": 1, - "0.1": 1, - "1": 1, - "10": 1 - }, - "count": 1, - "sum": 0.000200496 - } - } - } - } - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 1, - "0.032": 1, - "0.064": 1, - "0.128": 1, - "0.256": 1, - "0.512": 1 - }, - "count": 1, - "sum": 0.009125225 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/apps/v1/controllerrevisions?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 0 - }, - "count": 1, - "sum": 0.684430812 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/rbac.authorization.k8s.io/v1/clusterrolebindings?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 52, - "0.002": 739, - "0.004": 3489, - "0.008": 7832, - "0.016": 7920, - "0.032": 7920, - "0.064": 7920, - "0.128": 7920, - "0.256": 7920, - "0.512": 7921 - }, - "count": 7921, - "sum": 32.97823027900004 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/extensions/v1beta1?timeout=32s", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 0, - "0.128": 0, - "0.256": 0, - "0.512": 1 - }, - "count": 1, - "sum": 0.298650755 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/batch/v1/jobs?limit=%7Bvalue%7D\u0026resourceVersion=%7Bvalue%7D", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "node": { - "collector": { - "count": 1, - "eviction": { - "count": 0 - }, - "health": { - "pct": 100 - }, - "unhealthy": { - "count": 0 - } - } - }, - "process": { - "cpu": { - "sec": 2461 - }, - "fds": { - "max": { - "count": 1048576 - }, - "open": { - "count": 17 - } - }, - "memory": { - "resident": { - "bytes": 108044288 - }, - "virtual": { - "bytes": 788058112 - } - }, - "started": { - "sec": 1655707927.95 - } - } - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "duration": { - "sec": { - "bucket": { - "+Inf": 7921, - "0.001": 24, - "0.002": 574, - "0.004": 3334, - "0.008": 7842, - "0.016": 7921, - "0.032": 7921, - "0.064": 7921, - "0.128": 7921, - "0.256": 7921, - "0.512": 7921 - }, - "count": 7921, - "sum": 33.281374038000045 - } - } - } - }, - "url": "https://172.18.0.2:6443/apis/autoscaling/v1?timeout=32s", - "verb": "GET" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "client": { - "request": { - "count": 313 - } - }, - "code": "200", - "host": "172.18.0.2:6443", - "method": "PATCH" - } - }, - "metricset": { - "name": "controllermanager", - "period": 10000 - }, - "service": { - "address": "127.0.0.1:55555", - "type": "kubernetes" - } - }, - { - "event": { - "dataset": "kubernetes.controllermanager", - "duration": 115000, - "module": "kubernetes" - }, - "kubernetes": { - "controllermanager": { - "name": "garbage_collector_attempt_to_delete", + "name": "node_lifecycle_controller", "workqueue": { "adds": { - "count": 1592 + "count": 317 }, "depth": { "count": 0 @@ -8115,32 +7152,8 @@ "longestrunning": { "sec": 0 }, - "retries": { - "count": 0 - }, "unfinished": { "sec": 0 - }, - "work": { - "duration": { - "sec": { - "bucket": { - "+Inf": 1592, - "0.00000001": 0, - "0.0000001": 0, - "0.000001": 0, - "0.000009999999999999999": 0, - "0.00009999999999999999": 0, - "0.001": 0, - "0.01": 50, - "0.1": 1592, - "1": 1592, - "10": 1592 - }, - "count": 1592, - "sum": 20.110782428000025 - } - } } } } diff --git a/metricbeat/module/kubernetes/controllermanager/controllermanager.go b/metricbeat/module/kubernetes/controllermanager/controllermanager.go index 51243c37366..9f7d487ceef 100644 --- a/metricbeat/module/kubernetes/controllermanager/controllermanager.go +++ b/metricbeat/module/kubernetes/controllermanager/controllermanager.go @@ -25,17 +25,18 @@ import ( func init() { mapping := &prometheus.MetricsMapping{ Metrics: map[string]prometheus.MetricMap{ - "process_cpu_seconds_total": prometheus.Metric("process.cpu.sec"), - "process_resident_memory_bytes": prometheus.Metric("process.memory.resident.bytes"), - "process_virtual_memory_bytes": prometheus.Metric("process.memory.virtual.bytes"), - "process_open_fds": prometheus.Metric("process.fds.open.count"), - "process_max_fds": prometheus.Metric("process.fds.max.count"), - "process_start_time_seconds": prometheus.Metric("process.started.sec"), - "rest_client_request_duration_seconds": prometheus.Metric("client.request.duration.sec"), + "process_cpu_seconds_total": prometheus.Metric("process.cpu.sec"), + "process_resident_memory_bytes": prometheus.Metric("process.memory.resident.bytes"), + "process_virtual_memory_bytes": prometheus.Metric("process.memory.virtual.bytes"), + "process_open_fds": prometheus.Metric("process.fds.open.count"), + "process_max_fds": prometheus.Metric("process.fds.max.count"), + "process_start_time_seconds": prometheus.Metric("process.started.sec"), + // rest_client_request_duration_seconds buckets declared in + // https://github.com/kubernetes/component-base/blob/3b9b201c27aa896b98da61b94545efe442ae597e/metrics/prometheus/restclient/metrics.go#L39 + "rest_client_request_duration_seconds": prometheus.Metric("client.request.duration.us", prometheus.OpMultiplyBuckets(1000000)), "rest_client_requests_total": prometheus.Metric("client.request.count"), "workqueue_longest_running_processor_seconds": prometheus.Metric("workqueue.longestrunning.sec"), "workqueue_unfinished_work_seconds": prometheus.Metric("workqueue.unfinished.sec"), - "workqueue_work_duration_seconds": prometheus.Metric("workqueue.work.duration.sec"), "workqueue_adds_total": prometheus.Metric("workqueue.adds.count"), "workqueue_depth": prometheus.Metric("workqueue.depth.count"), "workqueue_retries_total": prometheus.Metric("workqueue.retries.count"), diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go index e6a30a0e0ec..5f3863d6747 100644 --- a/metricbeat/module/kubernetes/fields.go +++ b/metricbeat/module/kubernetes/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKubernetes returns asset data. // This is the base64 encoded zlib format compressed contents of module/kubernetes. func AssetKubernetes() string { - return "eJzsXV9z4zaSf59PgfKTc+Woru5x6mqrNp7dW18yE5/tSR6urhSIbElYkwADgPZoP/0VwH8gCYCkCMoeW3pIZWyr+4dGA+huNLp/RI9w+Ige8w1wChLEB4QkkQl8RBc/1z+8+IBQDCLiJJOE0Y/oLx8QQqj5A5SC5CRS3+aQABbwEe3wB4QESEnoTnxE/3shRHJxhS72UmYX/6d+t2dcriNGt2T3EW1xIuADQlsCSSw+agY/IopT6MBTH3nIFAfO8qz8iQWe+tzQLeMpVj9GmMZISCyJkCQSiG1RxmKBUkzxDmK0ORh8ViUFE42JCGdEAH8CXv/GBsoDrCO/v97eoIKgIcrq0xZp9elCM+Fx+DMHIVccBMt5BK0/qpA+wuGZ8bjzOw9e9bkrKEOMrLS7AES+WRKDi3wPRsSy8ACQJosuoyQXEviVZioyHMFVLZ0fvLiegG/CwfrHw8Mt6pHs8oxYHFAUmmePZJ8nlUDlWjEKPw0lBs0C9Vh0scT8sOY5DQfjd5B74EjuoeKBcgECxfyAuoy6YB4J7XKbgeRnQmO1r5XUB6YkzRgFKsOxv65Ioj2mcULozhSKF01315yJRG2nmiTasmpmRmwTT8AFYQFVoyRYo+gPswtBS651rMyEUC0SG+Eu8xTkngXUR70wLUR7g2YioBrWI+5SrdhmnEUghJWjTRFtJ61JL8rylYCo9/uKZszyTdLd93oDub79igREjMZdZA2nFFLGD+pYJzFQudocGpuozzdhdGf5ZWERfUSuL7dQ/aT+CBGKKp4lhiGIT4TLHCenRFiyHAK4jcWKZUBXEct7u98gtBbrL3m6Aa52XEUQbUkC9R8w7p5GITGXEAdQmvtCYZAgNAK9xZTKXfGwLoBnLKN9MPWHJ6BSrAT5FxTTvdrk0SPI1b85B8c2/4TIJvviF+vxU/C7GkoBASkEKCZCcrLJtclPqEOH3NhFni6qrvd5qhTmucEtNHBxDNiQKmwiGoJgMVvQwKbd49zfuJHevItzGj2WtozSaQOa4xwRGaPCbloeo9Ivo8tOiejBedRb2xeAo30x2KvK7tD/s2mckSvTYboq3RflFtfG4GqMSE60RKpZHb0+FloYDQ7JKpNFWPUwSoiSYWNi90E4ATiZF/wQFiV5nyUVzqQJKj+LyExmcc51hGaVH6VbLbaV8VfRVCqfKtVJScTZkH1lIpkvgj4WahoMxQ/HgTnhLlTvNwmWQKNDa8u5QnsiJNtxnKICkxt/lHOulsN8Qd7QbUJ2ezmsSooazykldBd4D6iWYSTJE+hvo5KRf08AGcWrYhKCbAhNjLKcWoGw1Fys7HEeE7nSR2cQ9pqezUpoM+SgoEEckGdFssu82bKoxITOC8ka0q3pBYnIanN8LUlqN1JiLLu/GDAN7hVB1CNo+KSjD4OhsM7tV5QLvAOLIFzDNqHo7zrXoQ2Qj2prkIzbCA8TH2JgMrHs0F02Tl+t+oww86rPda12Su7XjEMpfIqp8/xq4cWUKcG4YI+APBJuoRgQD7CsgbEYVpn1kGpwiQgnEK+3CcOuP6xsyQx41I9fHjUGJV8sEK5oqn+XvodkEicaO8JJwiIs8SYB9T3vYBOSEvn9jTaGLaEQF/DrsGWzFV6qnzglgsgW5VR/F2L7zUfCduMDbwOj+oXtlFewZRM3JPyESYLt6j9/U3K5OGjc2hvylND42dbyqQeLIpzhiMjDsCNV/eV7kE+hzeNloza89yAXvbGPFwtR+4E70DvvlLfb8SjoYfag9aBZLc4BGZFkDn7zIxwuxWoMJId2LgFJK4gFUvsiIFhE4r1s2l09HLjNWM64fm0iKQThHPArtzM/G+gnmpoODUCv3tocM+YZBmepEG6b05QQ7933oje2Su7u7/1rpL79Y/yR0J0Ad3DsbUjk92KgSIAcv3u8zqXkGspLL6sM72CL88QSkp12kW0fehMDVIyQg1NtceB/Mn4yRJqbE1e97zAmtwFTTd6HZ3bHmNTpFOIgJKSTnbT3Yiza5WQ6MWdv1i6j0nt5Oa/2JJ7aV4uPZt6XcJYkwIsM+Fn3Jtc1sTKfPsytSc4T69Y5Ky+wS7RiFjYZu+LmzMc+ZR72qRM7T5zQqf4bjt0XnMK4vOF/MRqQ7w3dciwkzyOZc+gTP6evFsM5p6++cPqqgpDibwEQfMbfSJqn7byY7yiR1pKRNWcl+hLHpgl2yHkZSCuzgarTkQRE32U+knUgi+WbDe2FVjQvknNmxoX+zCEPl0arAIM60YqErPmr9JeCYJ15Va5R3xaR0y2hROyD7BJfa2JjWOM4DpFG+Hs1L4qg4zaszhmETO6D8tQUB7hyZeoHybpt+FJjr9LUndyV3oZd1LfF3OoHbCTVQbV6cSAiIR29yvvY5stoBLpJeaZ9jCfc3UcMxr/Dt259IuWCRpLZ388d97iFRFouIc0tHXOtKPu2rj3gRO4PQZnXVJE9uLFECrifU4HHEXOfoEqtKLh7kPXpBDgGviJinWIhHW8uN4wlgLtPQodeBO+bJ8F6rolAHR69cEz3dcvkEMzDHsyKAuXTojIKAxKVa6P+jdxjiTAHtAOq9oiiBEKVU1yesC0OhCofSgn3525BBjQhvONWMMdce6V9ragUXBCHiPFYFHKvlU/tK8XPMswlifIE8/LF2B4LxCKdqB5bEOpvSpxmFpT9zcQXyNoSLuS6ZEUdxRCmJwE/VADVODUP1PBQP3O/mUrw4oAUiwE8jdstejdzBQYJ3+R4bfhc0Ck1AeLm5TV5AmoRR8Syw1oyG4LGusGi8yjcHeXxorvTlMaCq7WwW9HgSO4Ph6y+H/RztIS8XErv56jvHKuiABwyxmVRFYAIy1z4FtCi5Qq2nKXoeU+ivRZOsTcQ0eyMVkhhg5xf1DmhCCNGx2Ixwrs4xhLPn7HPJSWEhWAR0afCM5F77xryzZt9C51ukdV6wKE3Ici3YY0IjLQ2Lc2AMOpfKQ2gal7WYYPQ/1WSLVVi2yiD3foNHwEfxVNXuwnLWJNUvlWxCIoF8IyHVmMVqF8HL+LxW1nEwxSI/14gJwHvWr5S8qfyhWKgkmyJMiuZAcTiitXbOCTbdULoY0Awd7+ofZyDUGjKAi+uY4TQJ5Y8Qby2YFxqd6p42uTi26dwRsJrzl9vb+oSMKX2eKYrbC0gxdt8Q+9hHHbzMDcsD9Pl1mtFeYLowy7YrzefBnibQYs5Pp/xXFH7meeXiueXio5P+JeK2mL93h8pnl8r2P/m/Fqh9wn3WuGclN6DfE5Kd0E/p1gPpFhTkEp7gu3d/NsbV8E7iIA86Xi/LiNCqycDSsaIUAl8iyNYoZtt/6eIKFtTVi8MrtAzSRK0gTK8pw5aJpW9wyGS6AknOaA//v0Pr2iAc9s9+mjZjB33t5LTCw25jom9dQV74JiKlEj5/nTs4QV1rL5sOr83qT4jZ+3v56cmk0V0fmVifnrieR8PTIy0Ekc5gC6s01RzaHC9ljoODSJXLYfa/sypM/p2zB5OUmWvL1Sbw30+DDMYYoJGrnQ0Prw1ZsWjaWGwm1T7J9NPEDTyFEHvW5Ajzhk0Zdt7l0K0n0Z1iKH1jGvOLUTG4u/yEuIcQ6g+R8UQXoPnVXv3Z79bvqZ5ebDOy7u6/Xs1t109YK+xVNKUkpzvqgynOlzrEi6iW8OlrL/JKCDGUco4mH9cElYkMIehKp2Bb0PP13494K9y5Z2LlIVbjkeXVHovAcXWgnEPunNBvH77N8SFYJ5798Tf4eIKU7ZsoYX29rMvCk2qS1+oWdBNhAaydDK8g/WCF/wFrNHpBuvT4HEnGxiFV74d5gRKjOd5mtb8lr/1WxRLd8ejHym5GkY2Ifw4yIMkW80i40lLt7jQHC49cs5HPHOl1qZnqE6vZs+Ud4Qth9layMC7DEZW6mmb/0N1ejwvc/172DEVejrQ/PV5AiPzVuapL5H8dXk8kGbU5GmFSx0VLsYrxpQaOPX6kfL417Geaii5WJXmAUnAUonAW4egVYVgzMi7VVNa3cJQA8Qlfxt8WxGKUIiiPM0TrDtR9dn4QM1TzgpUVLy5ZryGNwDB6MwXaFbHg9UWz/gpHOhzGGZraWGbNJmDPQ6Pn0tF2sm9qMH0QhNpNsAcM5N9sEtOpYlu5Fz2Ac6dzBKEjUxXhQLqjc04HOqBeUyRh5GFy+rj8ECjUYeSl+ljvoHCTC+N9QONrFcLA0dbnoAYeTIMi//+QKNbBedOke2UHWPb+gdioOelG9089XDiG1GSyI3JWZAo5D7jhD5UYq5ziZxx/ccpobtg0/6lII0M2pNanI6EONN29YKcoAADKE+iDf7BuFWiFzUQ0R7iPJlXX9mIHNT0zmGDPo83Fjbovcw+ks1QSWPDMsmTIAO7L7UUYSkhzWSfdMWz3g0CslWL1Ub3HI45h2OGIJ3DMedwzERE53DMORxzDsecwzHncIwVg7fQasHfVmbVC2FKidWeL9YtbHrcIQn/Aad3S/9GYyQZAhobg7EfSyNhzwlLTEDjWYBdRPNWhB2TbyVmLF5lHJSbohDouszp4HwOI7llMWroopLuNBBzZsfO3zMRDgzz5sOBYmhCSr8xBPeKlJenWQBd6esp7d17y9oZPlh7iOeZuDYQo87PHo6ZEWTXov3QZVznhH3ocjnu8VLTHjPEE6ajS8r1xHNd4yL22pdCYpmHe/Sf7bFwZ5/aB9AdhC/7vR6OZoQuy1rjV+gZE6n/RwJPCcX+9D3Asbsugb1u+0iUDULNxC7flgGpHHJ3chqhEna9AvNHgCn4DHbj6NWrNsHMmr8HHT/UVazbDWWJ8oI5ByoT/eS1nEp0WcO/1oV11execyz2vzCW/YSjR7bdXqG/ca4fLt7mSXJlZVz/uvzOD4hxQ00UnzRLQEJ81UjsGlPK5F1ONQfGr9Cvv37+mSQJxD/oSYWVU4q6PnrDYL20VHWxdKton7EOexmD1dDHDVmJqRn0yrp9THmDNdjyR+chu14eFXRd6ceTFsP17VddVVEULD2rofJ8TgKpZAcxOlmtwULky3eqGxp+mXxd5G4PVr6o5uXlcTdTVqWPu95JR5zRf7JNKHOjoBbE2OhdFI03N9B1iaNHo3uDOZeBlY5hzJWNIuwrYwyfhgTKWEI6lOoHNpEyaK1WqlPpHJcCBSnlr4imyXpPSQybUKxFLjKgca9Igc9saXE3Ix+VChHl1dnoNpqry+ZbbiQ8DkLbrc5YtEeidydRQVCnlK04f+s8rTQgGA4ldH1yVjB4Tu0LBL4txF5RHmQfA44TQt2ch3TuU0mgZo23Eni9pDSSiOkGPVzZXVtMEmMmxvyP/59uNyzGkDLafpc1J8Hhk6Z3r58MnXhnbA6nLCERHu9SDRw41tGVTI58rT78TPD4PqEPrf5JTQGWSiwoA94MxAkxBkG4p/LXPIAl9VajwEnw/A5kKOkVHuREaDk93fQavIZgNhObJeyQzmxYZphCDcEgaz7DlipAow9Y7/I1kBZcbMGBE2wjBo560gjdsom7yNASnRW5+NRgbNStWrY16kuRQTTnOXQojP1lMGt9hoJlW59uYFlsbY8UHFTBpw+oAjLGTbIv7HFGgt1xWspFsjS627dbM2njz9cTSPkIofaB26bKVu12dF2OERaEdl4WOF+s/pETRsdSXQJGwcIPQ+RRBND3y8Ii0VyE2OZJH02FZFLxx/EnhtLQuhvbVGvT1XatdpxsjddGGyO68y+WRts7n0fZwlWFQE+KzM61g8mS3x5qGhsOSICUhO6mzueypnnE6Jbscq6joDXUJrWpkuPlfe/ob0w4jpMEEiK6146hhGhwePVSNLEaB45HfuyZOjoaz5ecpq2tTZ7a0g2HxGY5iVG4ixTzVK5bgelmx+yZiqL3aP+wbNBZ2rAFRFe1ZPMgQ5ewW6GLa87of7PNhds0JmIdMSo5S7pvOIJB/vW5utyrGaHLC8lzuLhCF1ucCPU/jKOL/6SMwl8u7No48YJ6mjoWxGfoY7WdLyNCM17bOjwcgszpI2XP1DPvAxZTULSl6TQWar3EX2dju5BZEv7w0axJ0J3VzNQDdKnEf4W08JXoS8m7dSSnZWjY66POypLQKFt8hhIlivu1dcZBiNzaEC+U8Iqqbrclo6OlGBPxeAq4n4h4nA2W5XLNtmuFeUGov+by163CezTOjMSnkOntzaejRLpEVoRRo2y5RIS6H6NZEc3OrR5jWaR+ANQMz7fCVJfDP10LRkPmL5Vk8MVVoG4oO6KemBdFXqL1tzIwAlyLzJ0u9beUappz4w0P1ROyKBxd6bDbUqMWMHBBhAQqn1iSp6Hsq4YsKug2oUXOUv2XP+qEsx9fOlHltwKeIuFIZPOtmnH37CUPa18TX37s1EEUqa84ihiPdR91ZsyJw3xlHO9gHSW4V7hyNPf7ggjSROpYTE+f0JhEAJdeRgkm6WLKGSX4Favo7W/XHv0shrCew+AnQmOIK2G4WZVJbetSa2asiLsmo7FaXuFXhZKbJmCnjXXUfJ12K0hM4PBXTQIpEnYeC66v29+uu4nG/VX06nqw7JmQa9I9ust7xONDNgqeIo1ubkO46dMYl47AcSk3C76L6MAsH0bcVQ8jboGqU2K1Wh37HiIkunmRiSplbaEAVnfCK242vFd9tN3UjXlZbcaqrLK9RKC8tgVzS0yo7iSX15Si1spbuiv+8XKZacfjerGUtBHY2EZXq15KaDugZb2emhPaHPTx2YBDnvuOJtF4A8skFLSluM2T5FBxG5Sm8VZM3/v8mTOJg20tBs0gm8ty6eJ3Jdb/0ViHksa7UpqCoOBQ3ApBjC73mMf6gBIQ/+ArhhXGE2gP1Pm2QtE7loU5wmLlqK9eoT/UUP9QY/1DDdbeJNo68CPGV9y3KlEW6oezLCEgkGR9j9H/T7eHqbYDEoUKeJTUXvzdzX2JwxPPSHIhgbuM8BE8bqgETnGCbm5rlS/Hb2cJ34ovzHJSq5FVxNCnL/fuJVCzPH6YPYYO3yJhOF5vcIJpNEusvzAco59KOrVCOZjOWeLVwHo06hwAuuPKNZ6jIpqCC33FQLlsc3SiYvMPGx1/mpkzkcsqKk1DbYatL5jeJWzzJJxhX1EMZtn7hDAUrLEnqtYiqfP/0CWoA7o4B+/LEXStvxO4Gi3h1TbUUd7GwvapUb62Mk9bNp9LiOgF3I5ejv1YgC/mfwypYOMdLK2Ehh9yfEbRwrpYa6AB9nXoYKV5I4B14q7dsOu8LdmMwr64nddC47b2Ms6eiCDMlbg54XKpodRYfSYK152BvrpZW95OT3IMNJXyBXZRlvJAcUoirBzm8nQrbzDsV13lPcmG6KjnrLD/ZxYXj2Nj0F29GtkQukOYxqjkEt4eaU37gFWie7WH0v6i8bvRMS+IVWIptjRpJiwNm+sSDE6D8NR94N9FO+qIWROvhokPMTCZ9B4/9Nl42yCi8QWlUCnta8ahFDnF1FEGsYPytXTmXigJ6tx5uQX67bf8vLu/HyeKsr/s22+n6+pB65RMhnewYLfR5jHg6A6oJ0M03AM1aOpZO9/sxQz0vlSMTDMrt224ZwTWTLMhoj7C6FWuwr+TBErDtOjd7E8qRZOukd+oiJp3/4MystS7QG9dPNqackrGHMRra4t+22p8XiZYaG/ROcGEstiTCD5nih26g4Kav1+LyXIMwthZOfh9kxBg/s4BxoCx11kNjaawX0fA+Q7UuBzF/wcAAP//jlv/BA==" + return "eJzsXU9v47iSv/enIHLKLDLGYo+NxQPepN/sy07/ySbpmcNi4aalss2JRGpIKmm/T78gRUmUROqPRTvuxDkMphO76sdikawqVhV/Ro+we48e8xVwChLEO4QkkQm8Rxe/Vb+8eIdQDCLiJJOE0ffob+8QQqj+AEpBchKpb3NIAAt4jzb4HUICpCR0I96j/70QIrm4QhdbKbOL/1N/2zIulxGja7J5j9Y4EfAOoTWBJBbvNYOfEcUptOCpH7nLFAfO8sz8xgFP/dzQNeMpVr9GmMZISCyJkCQSiK1RxmKBUkzxBmK02ll8FoaCjcZGhDMigD8Br/7iAtUDrCW/v9/eoIKgJcrypynS8qcNzYbH4a8chFxwECznETQ+VCJ9hN0z43Hrbz141c9dQRli5KTdBiDy1SEx+Mh3YEQsCw8AabLoMkpyIYFfaaYiwxFcVdL5qRfXE/BVOFj/fHi4RR2SbZ4RiwOKQvPskOzypBKoXCpG4afBYNAsUIdFG0vMd0ue03Aw/gC5BY7kFkoeKBcgUMx3qM2oDeaR0Da3GUh+IzRW+5qhPjAlacYoUBmO/XVJEm0xjRNCN7ZQetG0d82ZSNR2qkmiNStnZsQ28QRcEBZQNQzBCkV3mG0IWnKNY2UmhHKRuAi3macgtyygPuqF6SDaGTQTAdWwGnGbask24ywCIZwcXYroOmltelGWLwREnb+XNGOWr5L2vtcZyPXtVyQgYjRuI6s5pZAyvlPHOomBysVqV9tEXb4JoxvHHwuL6D3yfbmB6hf1IUQoKnkaDEMQnwiXOU6OidCwHAK4jsWCZUAXEcs7u98gtAbrz3m6Aq52XEUQrUkC1QcY90+jkJhLiAMozX2hMEgQGoHeYoxylzycC+AZy2gbTP3hCagUC0H+BcV0L1Z59Ahy8W/ewbHVnxC5ZF/8YTl+Cv5QQykgIIUAxURITla5NvkJ9eiQH7vI04Oq632eKoV5rnELDVzsAzakCtuIhiA4zBY0sGl3OHc3bqQ37+KcRo/GllE6bUHznCMiY1S4Tct9VPpldNkrET24HvXW9gXgaFsM9qq0O/T/rGpn5Mp2mK6M+6Lc4soYXIwRyZGWSDmro9fHgRZGjUOy0mQRTj2MEqJkWJvYXRBeAF7mBT+EhSHfZ0mFM2mCys8hMptZnHMdoVnke+lWg21p/JU0lcqnSnVSEnE2ZF/ZSOaLoIuF2gZD8ctxYI64C1X7TYIl0GjX2HKu0JYIyTYcp6jA5Mcf5Zyr5TBfkDd0nZDNVg6rkqLGc0oJ3QTeA8plGEnyBPrbyDDq3xNARvGimIQgG0IdozRTKxCWmouTPc5jIhf66AzCXtNzWQlNhhwUNIgD8ixJtpnXWxaVmNB5IVlLuhW9IBFZbY4vJUndRkqMZfsPA6bBvSKIOgQtn3T0YTAU1rn9inKBN+AQhG/YNhT9Xe86dAHqo9oYJOMuwsPEhxjYTBw7dJuN11crf0aYeeXPdaV2Su7XjIMRPsXUe3418GLKlGB8sEdAHgm3UAyIB1hWwFgMi8x5SNW4RIQTiJfrhGHfB0tbMgMedeOXe41ByRcLhEua6t/G95BM4kRjRzhJWIQlXiWgvtc72ISkRP54o41hTSjEBfwqbFlvhZfqN16JILJGOdXfhfinBbpZt76uPqP/LBDmgFIihDpAlQuiPvhNEf2m//lNSCxhWfzC7DtgvrZicqusEsU2RowiucVSA7pCckvK20j0TJIErWo2QCXhkOwWzh0zYZvxIcEBeX9kG+WvrNnErRI/YZJg98Kcv136nC80blcY8uHQeD3U8qkGiyKc4YjI3bCLV37yLcinWGfjZaO24rcgF33kjBcLURuDPwQ9z/5wexgo6DH7oPWgXi3eAVkxbg79hlE4XIrVGEge7TwEJK0gDkjNK4pgsZK3smm39XDgnuVwZv+piaQQhHfAJ24Bf7LQTzSCPRqATt4OHjPmGaawUYgha9igODWD2J4+3rkmR69sCd/d3/cv4OrSlPFHQjcC/DHF1yGRP4qBIgFy/NZ2muvcN5QjrXmvLmV4A2ucJ45I9rT7f/fQ69CpYoQ8nCpzCP/J+NEQaW5eXNW+w5hcB8zQeRtu4x1jUmehiJ2QkE72IN+KJeuWk+1hnV1tt4yMa/VyLvdR3MivDgfSvmbiLEmAF4UDs66britipgwhzGVTzhPn1jkrnbJNtGQWNoe95OZNYz9m+vqx82GPnAer/huO3Wecwrh0638xGpDvDV1zLCTPI5lz6BI/Z/0Wwzln/b5w1q+CkOLvARB8wt9JmqfNdKIfKP/Ykcg2ZyX25dtNE+yQ8zKQjecC9aNncbnG8bJJei5EL5KsZ0eG/sohD5d/rACDOtOKTLb56/RjQbBKWTOrtG+TyOmaUCK2QfaJrxWxMaxxHIeY0j/KeVEEPZd1VbIlZHIblKemOMCVK2M/SLpyzZdau5Wm7rb9WAyLSLkekWTucrP9akFIpNdkyGNWx9pKyn0KuwWcyO0uKPOKKnI7tYfImO7nVODxxFrHs7ttRD/9g6z2JMAx8AURyxQL6SlRXDGWAG5XUA4V0G7rClo910SgFo+OG94uBpnsej9swS7AN5U41cUKMmuj+ou+UsEc0AaoOg2KjgFlCq7ZVxscCFW2sxLub+3+BWiCW+9XMM9c90r7WlEpuCAOEeOxKOReKZ8kKRS/yzCXJMoTzE2B1RYLxCKd1x07EOpvSpxmDpTdzaQvgLEmXMilYUU9vQOm58w+lADVODUPVPNQv/OXGCX44IAUiwE8tbslOjcyBQYJ3+V4bfhU0DGaAHFdqEyegDrEEbFst5TMhaA+07Bo1VD7vftedHea0lhwlRa2GwDsyf1hl1X3Qv0cHaEOn9L3c9R3TWUNPYeMcVkU0RPhmIu+BXTQ6v41Zyl63pJoq4VT7A1E1DujE1LY4NZndU4owojRsVissB6OscTzZ+yToYSwECwi+lR4JnLbu4b65s29hU63yCo94NCZENS3YY1wiBublmZAGO1fKTWgcl6WYYOP/2XIGpVY18rgtn7DRz5H8dTNYcIy1iSV71wsgmIBPOOh1VgGaJfBe178bnpe2ALpjwfnJGCM/Sslf+WAdNSSrIkyK5kFxBFhqbZxSNbLhNDHgGDuPqp9nINQaEw/FN8xQugTS54gXjowHmp3Knm65NK3T+GMhNecv9/eVB1TjPb0TFfY1jmKt11y3sM47OZhb1g9TA+3XkvKE0QfdsF+vfkwwNsOWszx+azqPu1nngv7zoV9np/whX3aYv3Ra/rOKfTuz5xT6Ds/4VLoz8nIHcjnZGQf9HNq7UBqLQWptCfY3s2/v3IVvIMIyJOO9+uuG7RMFVcyRoRK4GscgS7w6PwWEWVryjKz/KqqwSjCe+qgZVLZOxwiiZ5wkgP69u/fekUDnLtuT0fLZuy4vxtOLzTkKib22hXsgWMqUiLl29OxhxfUseqy6VxnUP6MnLVfzyUGk0V0ri6wfzrieRuFBVZaiadGvQ3rOC0Galyn0lygRuRrMFDZnzn1Rt/22cNJquz1AzWM8J8PwwyGmKCRKx2ND2+NWfFoWhjsJtX+yfQTBI08RdDbFuSIcwZN2fbepBDdp1EVYmiU78y5hchY/ENeQpxjCOXPXjGEU/C8Ku/+7HfLU5qXB+e8vKnbv5O57eoAO8X+PVM6WL6prpXqcK1ad4h27w7TrpJRQIyjlHGwP2wIKxKYQ7uNz/CXiKi/cxrtLQPf2J6vJjvAT3J3OHf3CrdljGrx9SPtDegNBmYbi9o/6NZF+/L137QXgnnu3Lf/gBtAmLZf8zYDv6/76rNYCk2qWkeoWdBvFw1kO2V4A8sDJkoUsEanbSyPg8eftGE1Lvm+mxNwssocNa35Lw1XNT2ORyX3LvbyvVNZX4XEQQq7XD1/rNKgdnOeOVw65LzFUHOl1qTX1/NmSj3mcLeb3mUwstPNtD43PRXO/XvYPh1uJvW3CYyst7PNyL42PZBm9LQZ00dmvGJM6SFTrR8p968y7m8lYswDkoCjUUtvm5ZGk5YxI2/3HGn0P0E1EJ/8x/ZmCYUoytM8wfoBrC6b6e1ZxipnCSoqatcZr+ANQLAeBAw0q+PBaotn/BQOPK8YZmtpYJs0mYNPK+4/l4q0l3vRxOiFJtJ+d3PMTHbBHnIqbXQj57ILcO5kGhAuMm0VCqg3LuNw6OnNfZpljGz8VR2HOxqNOpR6mT7mKyjMdGOs72jkvKIZONryBMTIk2FY/Pc7Gt0qOHeKbKtxF1tXvxADXbz86OaphxffiBc3/Zi8/dpC7jNe6EM92lqX8RnXH04J3QSb9s8FaWTRntS0bSTEmbZrL8gJCjCA8ija0D8Yv0p0ogYi2kKcJ/P6E1uRg4reOWzQ5fHKwgadCvc92Qy1BLYskzwJMrB7o6UISwlpJrukS57VbhCQrVqsLrrncMw5HDME6RyOOYdjJiI6h2PO4ZhzOOYcjjmHY5wYehvWFvxd7Wp7IUxpVdvxxdoNYvc7JOE/4Phu6T9ojCRDQGNrMO5jaSTsOWGJCWh6FmAb0bwV4cbUtxIzFi8yDspNUQh0f+t0cD6HkdyyGNV0kaE7DcSc2XHz75kID4Z58+FBMTQhxm8Mwb0k1cuzUkCjr8e0d+8da2f4YO0gnmfiukCMOj87OGZGkH2L9l2bcZUT9q7NZb8isPp5yRClYHu35uuI57rCRdw9RIXEMg/XPCHbYuHPPnUPoD2IviqCajiaEbo0Pduv0DMmUv+PBJ4SivvT9wDH/v4O7v73I1HWCDUTt3wbBqRyyP3JaYRK2HQa9e8BpuAz+JZFp++3DWbW/D3o+KHuBt58kJUoL5hzoDLRpcNmKtFlBf9aNyhWs3vNsdh+ZCz7BUePbL2+Qv/gXBeA3uZJcuVkXP3ZfOcnxLilJopPmiUgIb6qJXaNKWXyLqeaA+NX6MuXT7+RJIH4Jz2p4E/D1n3mawbLQ0tVN513ivYZ67CXNVgNfdyQlZjqQbsrUqbUsg0+mKPzkH0VXAVdX/rxpMVwfftVd6cUBcue1VB6PkeBZNhBjI7Ws7EQ+eFfehsafmq9oj7cQaScl5fHXU9ZmT7uqzePOKN/slUoc6OgFsTY6FwUjTc30LXB0aHRvsGcy8BJxzLmzIMb7pUxhk9NAmUsIS1KVYFNpAxap5XqVTrPpUBBSvkron6kvKMklk0oliIXGdC40+yhz2xpcLcjH6UKEeXVuejWmqufH3DcSPQ4CE23OmPRFonOnUQJQZ1SrkcOGudpqQHBcCih65OzhMFz6l4g8P1A7BXlQfYx4Dgh1M95SOc+GAIVa7yWwKslpZFETD90xJXdtcYksWZizP/0/9PvhsUYUkabdVlzEhw+aHr3umToyDtjfThlCYnweJdq4MBxjs4w2bPqf7hMcP93Nh8a71DVjWxKsaAMeD0QL8QYBOE9HdTmATTUG8/sTYLX70CGkl7hQU6EltPjTa/FawhmPbFZwnbpzIffLFOoJhhkzWfY0U1p9AHbu3wtpAUXV3DgCNuIhaOaNELXbOIuMrREZ0UuPtQYa3Url22F+lJkEPnDPMMLIRTG7jKYtT5DwXKtTz+wLHY+MxUcVMGnC6gEMsZNci/scUaC23E6lIvkeDBw23ziSht/fW8rKR8h1D5wW3crq9yOtssxwoLQzssBzhenf+SF0bJUDwGjYNEPQ+RRBND1y8Ii0VyEWOdJF02JZFITzfEnhtLQ6lW7qdam7/m6ynFyPWA32hhRsBCW1vOBfR5lA1cZAj0qMjfXFiZHfnuoaaw5IAFSErqZOp+HNc0jRtdkk3MdBa2g1qlNpRwv7ztHf23CcZwkkBDRvnYMJUSLw8lL0cZqHTg98mPP1PMy9HzJadra2uSpK91wSGyOkxiFu0ixT+XqSTXdnYc9U1E05+keljU6x3N2AdGVT9v1IEOXsFmgi2vO6H+z1YXfNCZiGTEqOUvaNRzBIH95Li/3Kkbo8kLyHC6u0MUaJ0L9D+Po4j8po/C3C7c2TrygnqaOBfEZ+lhu54cRoR2vbRweHkHm9JGyZ9oz7wMWU1C0xnQaC7Va4qf5QGDILIn+8NGsSdAv1NmpB+hSif8KaeEr0RvJ+3UkpyY03OujzsqS0CgbfIYSJYr7tWXGQYjc+bBgKOEVneduDaO9pRgT8XgMuB+IeJwNluVyydZLhfmAUL/k8sta4d0bZ0biY8j09ubDXiI9RFaE1aPscIkI1buWdkc0N7dqjKbZ/wCoGZ5vial6VuB4T1laMn+pJIPPvgZ1Q9kR1cS8KHKDtv9JCCvAdZC5063+DqWa9tz0hoeqCTkoHN3psP00SSVg4IIICVQ+sSRPQ9lXNVlU0K1Di5yl+pM/64Szn186UeX3Ap4i4Ulk61s14+7ZDQ/n+zB9+bFTB1GkvuIoYjzW79Eza0485ivjeAPLKMGdxpWjud8XRJAmUsViOvqExiQC+PQySjBJD6acUYJPWEVvf7/u0c9iCMs5DH4hNIa4FIaflUlqWxqtmbEi7uqMxnJ5hV8VSm6agJs21lHzZdruIDGBw981CaRIuHkccH3d/n7dTjTurqKTe8tmy4RckvbRbe4R9w/ZKHiKNLq5DeGmT2NsHIH9Um4OWBfRgmkKI+7KwohboOqUWCwW+9ZDhEQ3LzJRpqwdKIDVnvCSmwvvVRdtO3VjXlabtSrLbC8RKK/tgLklNlR/kssppag18pbuin+8XGba/rheLCVtBDa20t2qDyW0DVDTr6fihFY7fXzW4FDPfUedaLyCwyQUNKW4zpNkV3IblKZVK6bvff7KmcTBthaLZpDN5XDp4ncG6/9orENJ420pTUFQcChuhSBGl1vMY31ACYh/6muGFcYTaA7UW1uh6O3Lwh5hsXLUV6/QNzXUb2qs39Rg3Y9tOwe+x/iK+1YlykL9cJYlBASSrOsx9v/T72Gq7YBEoQIehtqL193cGxw98YwkFxK4zwgfweOGSuAUJ+jmtlJ5M343S/hefGGWk1qOrCSGPny+9y+BiuX+w+ww9PgWCcPxcoUTTKNZYv3IcIx+MXQqhfIwnbPEy4F1aFQ5AHTDlWs8R0U0BR/6koFy2eboRMnmny46/Wlm3kQup6g0DbUZNr5ge5ewzpNwhn1JMZhl3yeEoWCNO1G1EkmV/4cuQR3QxTl4b0bQtv6O4Go0hFfZUHt5Gwe2T632taV52rD5fEJEL+B2dHLsxwJ8Mf9jSAVr7+DQSmj5IftnFB1YFysNtMCehg6WmjcCWCvu2g67ztuS7Sjsi9t5DTR+ay/j7IkIwnyJmxMul2pKtdVno/DdGeirm6WjdnqSY6CpmArsoi3ljuKURFg5zOZ0MzcY7qsuc0+yIjrqOSvs/4nFRXFsDPpVr1o2hG70K4+GS3h7pDHtA1aJfvM+lPYXD+hbL+YFsUoczZYmzYTj4euqBYPXIDz2e/pv4lnviDkTr4aJDzGwmXSKH7psep9BROMbSiEj7WvGwYicYuppg9hCeSovnB8oCer8OnQD9Ot/8vPu/n6cKMz7sq//OV3fG7ReyWR4Awd8bbQuBhz9AurREA2/gRo09ayZb/ZiBnpXKlammZPbOlwZgTPTbIhoH2F0kqvwV5KAMUyLt5v7k0rRpGvkVyqiuu5/UEaOfhfotYtHW1NeydiDOLVn0W8bD5+bBAvtLXonmFAW9ySCz5lij+6goObv12KyPIOwdlYO/b5JCDC/coAxYNx9VkOjKezXEXB+ADU2o/j/AAAA//9df1Lw" } diff --git a/metricbeat/module/kubernetes/kubernetes.go b/metricbeat/module/kubernetes/kubernetes.go index dc09f5a4a34..a5d8f71c821 100644 --- a/metricbeat/module/kubernetes/kubernetes.go +++ b/metricbeat/module/kubernetes/kubernetes.go @@ -42,7 +42,7 @@ type Module interface { mb.Module GetStateMetricsFamilies(prometheus p.Prometheus) ([]*dto.MetricFamily, error) GetKubeletStats(http *helper.HTTP) ([]byte, error) - GetPerfMetricsCache() *util.PerfMetricsCache + GetMetricsRepo() *util.MetricsRepo } type familiesCache struct { @@ -86,7 +86,7 @@ type module struct { kubeStateMetricsCache *kubeStateMetricsCache kubeletStatsCache *kubeletStatsCache - perfMetrics *util.PerfMetricsCache + metricsRepo *util.MetricsRepo cacheHash uint64 } @@ -97,25 +97,18 @@ func ModuleBuilder() func(base mb.BaseModule) (mb.Module, error) { kubeletStatsCache := &kubeletStatsCache{ cacheMap: make(map[uint64]*statsCache), } - perfMetrics := util.NewPerfMetricsCache(0) + metricsRepo := util.NewMetricsRepo() return func(base mb.BaseModule) (mb.Module, error) { hash, err := generateCacheHash(base.Config().Hosts) if err != nil { return nil, fmt.Errorf("error generating cache hash for kubeStateMetricsCache: %w", err) } - // NOTE: `Period * 2` is an arbitrary value to make the cache NEVER to expire before the next scraping run - // if different metricsets have different periods, we will effectively set (timeout = max(Period) * 2) - minCacheExpirationTime := base.Config().Period * 2 - if perfMetrics.GetTimeout() < minCacheExpirationTime { - perfMetrics.SetOrUpdateTimeout(minCacheExpirationTime) - } - m := module{ BaseModule: base, kubeStateMetricsCache: kubeStateMetricsCache, kubeletStatsCache: kubeletStatsCache, - perfMetrics: perfMetrics, + metricsRepo: metricsRepo, cacheHash: hash, } return &m, nil @@ -167,6 +160,6 @@ func generateCacheHash(host []string) (uint64, error) { return id, nil } -func (m *module) GetPerfMetricsCache() *util.PerfMetricsCache { - return m.perfMetrics +func (m *module) GetMetricsRepo() *util.MetricsRepo { + return m.metricsRepo } diff --git a/metricbeat/module/kubernetes/node/node.go b/metricbeat/module/kubernetes/node/node.go index 8564d539af4..ddb918bb2d7 100644 --- a/metricbeat/module/kubernetes/node/node.go +++ b/metricbeat/module/kubernetes/node/node.go @@ -76,7 +76,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, http: http, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Node{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Node{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/node/node_test.go b/metricbeat/module/kubernetes/node/node_test.go index ae543b735a1..c2129c9b57b 100644 --- a/metricbeat/module/kubernetes/node/node_test.go +++ b/metricbeat/module/kubernetes/node/node_test.go @@ -25,27 +25,51 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) +// this file is used for the tests to compare expected result const testFile = "../_meta/test/stats_summary.json" -func TestEventMapping(t *testing.T) { - logger := logp.NewLogger("kubernetes.node") +type NodeTestSuite struct { + suite.Suite + Logger *logp.Logger +} + +func (s *NodeTestSuite) SetupTest() { + s.Logger = logp.NewLogger("kubernetes.node") +} +func (s *NodeTestSuite) ReadTestFile(testFile string) []byte { f, err := os.Open(testFile) - assert.NoError(t, err, "cannot open test file "+testFile) + s.NoError(err, "cannot open test file "+testFile) body, err := ioutil.ReadAll(f) - assert.NoError(t, err, "cannot read test file "+testFile) + s.NoError(err, "cannot read test file "+testFile) + + return body +} + +func (s *NodeTestSuite) TestEventMapping() { + body := s.ReadTestFile(testFile) + event, err := eventMapping(body, s.Logger) + + s.basicTests(event, err) +} + +func (s *NodeTestSuite) testValue(event mapstr.M, field string, expected interface{}) { + data, err := event.GetValue(field) + s.NoError(err, "Could not read field "+field) + s.EqualValues(expected, data, "Wrong value for field "+field) +} - event, err := eventMapping(body, logger) - assert.NoError(t, err, "error mapping "+testFile) +func (s *NodeTestSuite) basicTests(event mapstr.M, err error) { + s.NoError(err, "error mapping "+testFile) - testCases := map[string]interface{}{ + basicTestCases := map[string]interface{}{ "cpu.usage.core.ns": int64(4189523881380), "cpu.usage.nanocores": 18691146, @@ -75,13 +99,15 @@ func TestEventMapping(t *testing.T) { "runtime.imagefs.used.bytes": 860204379, } + s.RunMetricsTests(event, basicTestCases) +} + +func (s *NodeTestSuite) RunMetricsTests(event mapstr.M, testCases map[string]interface{}) { for k, v := range testCases { - testValue(t, event, k, v) + s.testValue(event, k, v) } } -func testValue(t *testing.T, event mapstr.M, field string, value interface{}) { - data, err := event.GetValue(field) - assert.NoError(t, err, "Could not read field "+field) - assert.EqualValues(t, data, value, "Wrong value for field "+field) +func TestNodeTestSuite(t *testing.T) { + suite.Run(t, new(NodeTestSuite)) } diff --git a/metricbeat/module/kubernetes/pod/_meta/fields.yml b/metricbeat/module/kubernetes/pod/_meta/fields.yml index 0b46d290839..89ef634c660 100644 --- a/metricbeat/module/kubernetes/pod/_meta/fields.yml +++ b/metricbeat/module/kubernetes/pod/_meta/fields.yml @@ -56,7 +56,7 @@ type: scaled_float format: percent description: > - CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited) + CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited). If one or more containers of the pod is unlimited and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. - name: memory type: group fields: @@ -77,7 +77,7 @@ type: scaled_float format: percent description: > - Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited) + Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited). If one or more containers of the pod is unlimited and the `node` and `state_node` metricsets are both disabled on that node, this metric will be missing entirely. - name: available type: group fields: diff --git a/metricbeat/module/kubernetes/pod/data.go b/metricbeat/module/kubernetes/pod/data.go index f090ccbc5e2..fdabbfab640 100644 --- a/metricbeat/module/kubernetes/pod/data.go +++ b/metricbeat/module/kubernetes/pod/data.go @@ -29,7 +29,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *logp.Logger) ([]mapstr.M, error) { +func eventMapping(content []byte, metricsRepo *util.MetricsRepo, logger *logp.Logger) ([]mapstr.M, error) { events := []mapstr.M{} var summary kubernetes.Summary @@ -39,24 +39,48 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *lo } node := summary.Node - nodeCores := perfMetrics.NodeCoresAllocatable.Get(node.NodeName) - nodeMem := perfMetrics.NodeMemAllocatable.Get(node.NodeName) + + nodeCores := 0.0 + nodeMem := 0.0 + + nodeStore := metricsRepo.GetNodeStore(node.NodeName) + nodeMetrics := nodeStore.GetNodeMetrics() + if nodeMetrics.CoresAllocatable != nil { + nodeCores = nodeMetrics.CoresAllocatable.Value + } + if nodeMetrics.MemoryAllocatable != nil { + nodeMem = nodeMetrics.MemoryAllocatable.Value + } for _, pod := range summary.Pods { var usageNanoCores, usageMem, availMem, rss, workingSet, pageFaults, majorPageFaults uint64 - var coresLimit, memLimit float64 - - for _, cont := range pod.Containers { - cuid := util.ContainerUID(pod.PodRef.Namespace, pod.PodRef.Name, cont.Name) - usageNanoCores += cont.CPU.UsageNanoCores - usageMem += cont.Memory.UsageBytes - availMem += cont.Memory.AvailableBytes - rss += cont.Memory.RssBytes - workingSet += cont.Memory.WorkingSetBytes - pageFaults += cont.Memory.PageFaults - majorPageFaults += cont.Memory.MajorPageFaults - - coresLimit += perfMetrics.ContainerCoresLimit.GetWithDefault(cuid, nodeCores) - memLimit += perfMetrics.ContainerMemLimit.GetWithDefault(cuid, nodeMem) + var podCoreLimit, podMemLimit float64 + + podId := util.NewPodId(pod.PodRef.Namespace, pod.PodRef.Name) + podStore := nodeStore.GetPodStore(podId) + + for _, container := range pod.Containers { + usageNanoCores += container.CPU.UsageNanoCores + usageMem += container.Memory.UsageBytes + availMem += container.Memory.AvailableBytes + rss += container.Memory.RssBytes + workingSet += container.Memory.WorkingSetBytes + pageFaults += container.Memory.PageFaults + majorPageFaults += container.Memory.MajorPageFaults + + containerStore := podStore.GetContainerStore(container.Name) + containerMetrics := containerStore.GetContainerMetrics() + + containerCoresLimit := nodeCores + if containerMetrics.CoresLimit != nil { + containerCoresLimit = containerMetrics.CoresLimit.Value + } + + containerMemLimit := nodeMem + if containerMetrics.MemoryLimit != nil { + containerMemLimit = containerMetrics.MemoryLimit.Value + } + podCoreLimit += containerCoresLimit + podMemLimit += containerMemLimit } podEvent := mapstr.M{ @@ -108,29 +132,47 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *lo kubernetes2.ShouldPut(podEvent, "start_time", pod.StartTime, logger) } - if coresLimit > nodeCores { - coresLimit = nodeCores + // NOTE: + // - `podCoreLimit > `nodeCores` is possible if a pod has more than one container + // and at least one of them doesn't have a limit set. The container without limits + // inherit a limit = `nodeCores` and the sum of all limits for all the + // containers will be > `nodeCores`. In this case we want to cap the + // value of `podCoreLimit` to `nodeCores`. + // - `nodeCores` can be 0 if `state_node` and/or `node` metricsets are disabled. + // - if `nodeCores` == 0 and podCoreLimit > 0` we need to avoid that `podCoreLimit` is + // incorrectly overridden to 0. That's why we check for `nodeCores > 0`. + if nodeCores > 0 && podCoreLimit > nodeCores { + podCoreLimit = nodeCores } - if memLimit > nodeMem { - memLimit = nodeMem + // NOTE: + // - `podMemLimit > `nodeMem` is possible if a pod has more than one container + // and at least one of them doesn't have a limit set. The container without limits + // inherit a limit = `nodeMem` and the sum of all limits for all the + // containers will be > `nodeMem`. In this case we want to cap the + // value of `podMemLimit` to `nodeMem`. + // - `nodeMem` can be 0 if `state_node` and/or `node` metricsets are disabled. + // - if `nodeMem` == 0 and podMemLimit > 0` we need to avoid that `podMemLimit` is + // incorrectly overridden to 0. That's why we check for `nodeMem > 0`. + if nodeMem > 0 && podMemLimit > nodeMem { + podMemLimit = nodeMem } if nodeCores > 0 { kubernetes2.ShouldPut(podEvent, "cpu.usage.node.pct", float64(usageNanoCores)/1e9/nodeCores, logger) } - if coresLimit > 0 { - kubernetes2.ShouldPut(podEvent, "cpu.usage.limit.pct", float64(usageNanoCores)/1e9/coresLimit, logger) + if podCoreLimit > 0 { + kubernetes2.ShouldPut(podEvent, "cpu.usage.limit.pct", float64(usageNanoCores)/1e9/podCoreLimit, logger) } if usageMem > 0 { if nodeMem > 0 { kubernetes2.ShouldPut(podEvent, "memory.usage.node.pct", float64(usageMem)/nodeMem, logger) } - if memLimit > 0 { - kubernetes2.ShouldPut(podEvent, "memory.usage.limit.pct", float64(usageMem)/memLimit, logger) - kubernetes2.ShouldPut(podEvent, "memory.working_set.limit.pct", float64(workingSet)/memLimit, logger) + if podMemLimit > 0 { + kubernetes2.ShouldPut(podEvent, "memory.usage.limit.pct", float64(usageMem)/podMemLimit, logger) + kubernetes2.ShouldPut(podEvent, "memory.working_set.limit.pct", float64(workingSet)/podMemLimit, logger) } } @@ -138,10 +180,10 @@ func eventMapping(content []byte, perfMetrics *util.PerfMetricsCache, logger *lo if nodeMem > 0 { kubernetes2.ShouldPut(podEvent, "memory.usage.node.pct", float64(workingSet)/nodeMem, logger) } - if memLimit > 0 { - kubernetes2.ShouldPut(podEvent, "memory.usage.limit.pct", float64(workingSet)/memLimit, logger) + if podMemLimit > 0 { + kubernetes2.ShouldPut(podEvent, "memory.usage.limit.pct", float64(workingSet)/podMemLimit, logger) - kubernetes2.ShouldPut(podEvent, "memory.working_set.limit.pct", float64(workingSet)/memLimit, logger) + kubernetes2.ShouldPut(podEvent, "memory.working_set.limit.pct", float64(workingSet)/podMemLimit, logger) } } diff --git a/metricbeat/module/kubernetes/pod/pod.go b/metricbeat/module/kubernetes/pod/pod.go index 1df68669fc1..21d74180f3c 100644 --- a/metricbeat/module/kubernetes/pod/pod.go +++ b/metricbeat/module/kubernetes/pod/pod.go @@ -77,7 +77,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, http: http, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Pod{}, mod.GetPerfMetricsCache(), true), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Pod{}, mod.GetMetricsRepo(), true), mod: mod, }, nil } @@ -95,7 +95,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { return } - events, err := eventMapping(body, m.mod.GetPerfMetricsCache(), m.Logger()) + events, err := eventMapping(body, m.mod.GetMetricsRepo(), m.Logger()) if err != nil { m.Logger().Error(err) reporter.Error(err) diff --git a/metricbeat/module/kubernetes/pod/pod_test.go b/metricbeat/module/kubernetes/pod/pod_test.go index 226433acf1a..d8e02c3b7fb 100644 --- a/metricbeat/module/kubernetes/pod/pod_test.go +++ b/metricbeat/module/kubernetes/pod/pod_test.go @@ -24,45 +24,78 @@ import ( "io/ioutil" "os" "testing" - "time" - - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/stretchr/testify/suite" ) +// both these two files are used in tests to compare expected result const testFile = "../_meta/test/stats_summary.json" +const testFileWithMultipleContainers = "../_meta/test/stats_summary_multiple_containers.json" + +type PodTestSuite struct { + suite.Suite + MetricsRepo *util.MetricsRepo + NodeName string + Namespace string + PodName string + ContainerName string + AnotherContainerName string + PodId util.PodId + Logger *logp.Logger + NodeMetrics *util.NodeMetrics + ContainerMetrics *util.ContainerMetrics + AnotherContainerMetrics *util.ContainerMetrics +} + +func (s *PodTestSuite) SetupTest() { + s.MetricsRepo = util.NewMetricsRepo() + s.NodeName = "gke-beats-default-pool-a5b33e2e-hdww" + s.Namespace = "default" + s.PodName = "nginx-deployment-2303442956-pcqfc" + s.ContainerName = "nginx" + s.AnotherContainerName = "sidecar" + + s.PodId = util.NewPodId(s.Namespace, s.PodName) + + s.Logger = logp.NewLogger("kubernetes.pod") -func TestEventMapping(t *testing.T) { - logger := logp.NewLogger("kubernetes.pod") + s.NodeMetrics = util.NewNodeMetrics() + s.NodeMetrics.CoresAllocatable = util.NewFloat64Metric(2) + s.NodeMetrics.MemoryAllocatable = util.NewFloat64Metric(146227200) + s.ContainerMetrics = util.NewContainerMetrics() + s.ContainerMetrics.MemoryLimit = util.NewFloat64Metric(14622720) + + s.AnotherContainerMetrics = util.NewContainerMetrics() + s.AnotherContainerMetrics.MemoryLimit = util.NewFloat64Metric(14622720) +} + +func (s *PodTestSuite) ReadTestFile(testFile string) []byte { f, err := os.Open(testFile) - assert.NoError(t, err, "cannot open test file "+testFile) + s.NoError(err, "cannot open test file "+testFile) body, err := ioutil.ReadAll(f) - assert.NoError(t, err, "cannot read test file "+testFile) + s.NoError(err, "cannot read test file "+testFile) - cache := util.NewPerfMetricsCache(120 * time.Second) - cache.NodeCoresAllocatable.Set("gke-beats-default-pool-a5b33e2e-hdww", 2) - cache.NodeMemAllocatable.Set("gke-beats-default-pool-a5b33e2e-hdww", 146227200) - cache.ContainerMemLimit.Set(util.ContainerUID("default", "nginx-deployment-2303442956-pcqfc", "nginx"), 14622720) + return body +} - events, err := eventMapping(body, cache, logger) - assert.NoError(t, err, "error mapping "+testFile) +func (s *PodTestSuite) TestEventMapping() { + s.MetricsRepo.DeleteAllNodeStore() - assert.Len(t, events, 1, "got wrong number of events") + s.addNodeMetric(s.NodeMetrics) + s.addContainerMetric(s.ContainerName, s.ContainerMetrics) - testCases := map[string]interface{}{ - "name": "nginx-deployment-2303442956-pcqfc", - "uid": "beabc196-2456-11e7-a3ad-42010a840235", + body := s.ReadTestFile(testFile) + events, err := eventMapping(body, s.MetricsRepo, s.Logger) - "network.rx.bytes": 107056, - "network.rx.errors": 0, - "network.tx.bytes": 72447, - "network.tx.errors": 0, + s.basicTests(events, err) + cpuMemoryTestCases := map[string]interface{}{ // calculated pct fields: "cpu.usage.nanocores": 11263994, "cpu.usage.node.pct": 0.005631997, @@ -74,13 +107,148 @@ func TestEventMapping(t *testing.T) { "memory.working_set.limit.pct": 0.09943977591036414, } - for k, v := range testCases { - testValue(t, events[0], k, v) + s.RunMetricsTests(events[0], cpuMemoryTestCases) +} + +func (s *PodTestSuite) TestEventMappingWithZeroNodeMetrics() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeMetrics := util.NewNodeMetrics() + s.addNodeMetric(nodeMetrics) + + s.addContainerMetric(s.ContainerName, s.ContainerMetrics) + + body := s.ReadTestFile(testFile) + events, err := eventMapping(body, s.MetricsRepo, s.Logger) + + s.basicTests(events, err) + + cpuMemoryTestCases := map[string]interface{}{ + "cpu.usage.nanocores": 11263994, + + "memory.usage.bytes": 1462272, + "memory.working_set.limit.pct": 0.09943977591036414, } + + s.RunMetricsTests(events[0], cpuMemoryTestCases) +} + +func (s *PodTestSuite) TestEventMappingWithNoNodeMetrics() { + s.MetricsRepo.DeleteAllNodeStore() + + s.addContainerMetric(s.ContainerName, s.ContainerMetrics) + + body := s.ReadTestFile(testFile) + events, err := eventMapping(body, s.MetricsRepo, s.Logger) + + s.basicTests(events, err) + + cpuMemoryTestCases := map[string]interface{}{ + "cpu.usage.nanocores": 11263994, + + "memory.usage.bytes": 1462272, + "memory.usage.limit.pct": 0.1, + "memory.working_set.limit.pct": 0.09943977591036414, + } + + s.RunMetricsTests(events[0], cpuMemoryTestCases) } -func testValue(t *testing.T, event mapstr.M, field string, expected interface{}) { +func (s *PodTestSuite) TestEventMappingWithMultipleContainers() { + s.MetricsRepo.DeleteAllNodeStore() + + s.addNodeMetric(s.NodeMetrics) + s.addContainerMetric(s.ContainerName, s.ContainerMetrics) + + body := s.ReadTestFile(testFileWithMultipleContainers) // NOTE: different test file + events, err := eventMapping(body, s.MetricsRepo, s.Logger) + + s.basicTests(events, err) + + cpuMemoryTestCases := map[string]interface{}{ + // Following comments explain what is the difference with the test `TestEventMapping` + "cpu.usage.nanocores": 22527988, // 2x usage since 2 container + "cpu.usage.node.pct": 0.011263994, // 2x usage since 2 container + "cpu.usage.limit.pct": 0.011263994, // same value as `cpu.usage.node.pct` since `podCoreLimit` = 2x nodeCores = `nodeCores` (capped value) + + "memory.usage.bytes": 2924544, // 2x since 2 containers + "memory.usage.node.pct": 0.02, // 2x usage since 2 containers + "memory.usage.limit.pct": 0.02, // same value as `cpu.usage.node.pct` since 2 containers but only 1 with limit, podMemLimit = containerMemLimit + nodeLimit > nodeLimit = nodeLimit (capped value) + "memory.working_set.limit.pct": 0.019887955182072828, // similar concept to `memory.usage.limit.pct`. 2x usage but denominator 10x since nodeLimit = 10x containerMemLimit + } + + s.RunMetricsTests(events[0], cpuMemoryTestCases) +} + +func (s *PodTestSuite) TestEventMappingWithMultipleContainersWithAllMemLimits() { + s.MetricsRepo.DeleteAllNodeStore() + + s.addNodeMetric(s.NodeMetrics) + s.addContainerMetric(s.ContainerName, s.ContainerMetrics) + s.addContainerMetric(s.AnotherContainerName, s.AnotherContainerMetrics) + + body := s.ReadTestFile(testFileWithMultipleContainers) // NOTE: different test file + events, err := eventMapping(body, s.MetricsRepo, s.Logger) + + s.basicTests(events, err) + + cpuMemoryTestCases := map[string]interface{}{ + // Following comments explain what is the difference with the test `TestEventMapping + "cpu.usage.nanocores": 22527988, // 2x usage since 2 container + "cpu.usage.node.pct": 0.011263994, // 2x usage since 2 container + "cpu.usage.limit.pct": 0.011263994, // same value as `cpu.usage.node.pct` since `podCoreLimit` = 2x nodeCores = `nodeCores` (capped value) + + "memory.usage.bytes": 2924544, // 2x since 2 containers + "memory.usage.node.pct": 0.02, // 2x usage since 2 containers + "memory.usage.limit.pct": 0.1, // 2x usage / 2x limit = same value + "memory.working_set.limit.pct": 0.09943977591036414, // 2x usage / 2x limit = same value + } + + s.RunMetricsTests(events[0], cpuMemoryTestCases) +} + +func (s *PodTestSuite) testValue(event mapstr.M, field string, expected interface{}) { data, err := event.GetValue(field) - assert.NoError(t, err, "Could not read field "+field) - assert.EqualValues(t, expected, data, "Wrong value for field "+field) + s.NoError(err, "Could not read field "+field) + s.EqualValues(expected, data, "Wrong value for field "+field) +} + +func (s *PodTestSuite) addContainerMetric(containerName string, containerMetric *util.ContainerMetrics) { + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + podStore, _ := nodeStore.AddPodStore(s.PodId) + containerStore, _ := podStore.AddContainerStore(containerName) + containerStore.SetContainerMetrics(containerMetric) +} + +func (s *PodTestSuite) addNodeMetric(nodeMetrics *util.NodeMetrics) { + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + nodeStore.SetNodeMetrics(nodeMetrics) +} + +func (s *PodTestSuite) basicTests(events []mapstr.M, err error) { + s.NoError(err, "error mapping "+testFile) + + s.Len(events, 1, "got wrong number of events") + + basicTestCases := map[string]interface{}{ + "name": "nginx-deployment-2303442956-pcqfc", + "uid": "beabc196-2456-11e7-a3ad-42010a840235", + + "network.rx.bytes": 107056, + "network.rx.errors": 0, + "network.tx.bytes": 72447, + "network.tx.errors": 0, + } + + s.RunMetricsTests(events[0], basicTestCases) +} + +func (s *PodTestSuite) RunMetricsTests(events mapstr.M, testCases map[string]interface{}) { + for k, v := range testCases { + s.testValue(events, k, v) + } +} + +func TestPodTestSuite(t *testing.T) { + suite.Run(t, new(PodTestSuite)) } diff --git a/metricbeat/module/kubernetes/state_container/state_container.go b/metricbeat/module/kubernetes/state_container/state_container.go index fd05f6f0d37..3bef1fde452 100644 --- a/metricbeat/module/kubernetes/state_container/state_container.go +++ b/metricbeat/module/kubernetes/state_container/state_container.go @@ -119,7 +119,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewContainerMetadataEnricher(base, mod.GetPerfMetricsCache(), false), + enricher: util.NewContainerMetadataEnricher(base, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go b/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go index 2e69ce47316..97b51ed0761 100644 --- a/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go +++ b/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go @@ -63,7 +63,7 @@ func NewCronJobMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, prometheus: prometheus, mod: mod, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.CronJob{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.CronJob{}, mod.GetMetricsRepo(), false), mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ "kube_cronjob_info": p.InfoMetric(), diff --git a/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go b/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go index de0956994f8..b3a56608764 100644 --- a/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go +++ b/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go @@ -89,7 +89,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.DaemonSet{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.DaemonSet{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_deployment/state_deployment.go b/metricbeat/module/kubernetes/state_deployment/state_deployment.go index 87bb67320e3..f7c8cdf2f0b 100644 --- a/metricbeat/module/kubernetes/state_deployment/state_deployment.go +++ b/metricbeat/module/kubernetes/state_deployment/state_deployment.go @@ -90,7 +90,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Deployment{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Deployment{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_job/state_job.go b/metricbeat/module/kubernetes/state_job/state_job.go index f49f75bf22a..26b1b619b7c 100644 --- a/metricbeat/module/kubernetes/state_job/state_job.go +++ b/metricbeat/module/kubernetes/state_job/state_job.go @@ -108,7 +108,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Job{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Job{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_node/state_node.go b/metricbeat/module/kubernetes/state_node/state_node.go index be48d50b1e9..0a5a84e8367 100644 --- a/metricbeat/module/kubernetes/state_node/state_node.go +++ b/metricbeat/module/kubernetes/state_node/state_node.go @@ -114,7 +114,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Node{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Node{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go index dc5e0db7ee2..ae85783c929 100644 --- a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go +++ b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go @@ -58,7 +58,7 @@ func NewPersistentVolumeMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, prometheus: prometheus, mod: mod, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.PersistentVolume{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.PersistentVolume{}, mod.GetMetricsRepo(), false), mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ "kube_persistentvolume_capacity_bytes": p.Metric("capacity.bytes"), diff --git a/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go b/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go index e257828233e..92abee69847 100644 --- a/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go +++ b/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go @@ -58,7 +58,7 @@ func NewpersistentvolumeclaimMetricSet(base mb.BaseMetricSet) (mb.MetricSet, err BaseMetricSet: base, prometheus: prometheus, mod: mod, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.PersistentVolumeClaim{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.PersistentVolumeClaim{}, mod.GetMetricsRepo(), false), mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ diff --git a/metricbeat/module/kubernetes/state_pod/state_pod.go b/metricbeat/module/kubernetes/state_pod/state_pod.go index f1981e7207e..11214e93e19 100644 --- a/metricbeat/module/kubernetes/state_pod/state_pod.go +++ b/metricbeat/module/kubernetes/state_pod/state_pod.go @@ -92,7 +92,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Pod{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Pod{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go b/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go index 1b730c54df4..50343428db5 100644 --- a/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go +++ b/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go @@ -90,7 +90,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.ReplicaSet{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.ReplicaSet{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/state_service/state_service.go b/metricbeat/module/kubernetes/state_service/state_service.go index f079f55ed08..5bda01acf12 100644 --- a/metricbeat/module/kubernetes/state_service/state_service.go +++ b/metricbeat/module/kubernetes/state_service/state_service.go @@ -86,7 +86,7 @@ func NewServiceMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { "hostname": p.Label("ingress_hostname"), }, }, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Service{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.Service{}, mod.GetMetricsRepo(), false), }, nil } diff --git a/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go b/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go index 7ad669c0830..149903613e8 100644 --- a/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go +++ b/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go @@ -90,7 +90,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - enricher: util.NewResourceMetadataEnricher(base, &kubernetes.StatefulSet{}, mod.GetPerfMetricsCache(), false), + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.StatefulSet{}, mod.GetMetricsRepo(), false), mod: mod, }, nil } diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index e7c918c5117..aeddde7dbbf 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -82,7 +82,7 @@ const selector = "kubernetes" func NewResourceMetadataEnricher( base mb.BaseMetricSet, res kubernetes.Resource, - perfMetrics *PerfMetricsCache, + metricsRepo *MetricsRepo, nodeScope bool) Enricher { config, err := GetValidatedConfig(base) @@ -121,18 +121,20 @@ func NewResourceMetadataEnricher( m[id] = podMetaGen.Generate(r) case *kubernetes.Node: - // Report node allocatable resources to PerfMetrics cache - name := r.GetObjectMeta().GetName() + nodeName := r.GetObjectMeta().GetName() + metrics := NewNodeMetrics() if cpu, ok := r.Status.Capacity["cpu"]; ok { if q, err := resource.ParseQuantity(cpu.String()); err == nil { - perfMetrics.NodeCoresAllocatable.Set(name, float64(q.MilliValue())/1000) + metrics.CoresAllocatable = NewFloat64Metric(float64(q.MilliValue()) / 1000) } } if memory, ok := r.Status.Capacity["memory"]; ok { if q, err := resource.ParseQuantity(memory.String()); err == nil { - perfMetrics.NodeMemAllocatable.Set(name, float64(q.Value())) + metrics.MemoryAllocatable = NewFloat64Metric(float64(q.Value())) } } + nodeStore, _ := metricsRepo.AddNodeStore(nodeName) + nodeStore.SetNodeMetrics(metrics) m[id] = metaGen.Generate("node", r) @@ -163,6 +165,13 @@ func NewResourceMetadataEnricher( // delete func(m map[string]mapstr.M, r kubernetes.Resource) { accessor, _ := meta.Accessor(r) + + switch r := r.(type) { + case *kubernetes.Node: + nodeName := r.GetObjectMeta().GetName() + metricsRepo.DeleteNodeStore(nodeName) + } + id := join(accessor.GetNamespace(), accessor.GetName()) delete(m, id) }, @@ -184,7 +193,7 @@ func NewResourceMetadataEnricher( // NewContainerMetadataEnricher returns an Enricher configured for container events func NewContainerMetadataEnricher( base mb.BaseMetricSet, - perfMetrics *PerfMetricsCache, + metricsRepo *MetricsRepo, nodeScope bool) Enricher { config, err := GetValidatedConfig(base) @@ -224,21 +233,28 @@ func NewContainerMetadataEnricher( } mapStatuses(pod.Status.ContainerStatuses) mapStatuses(pod.Status.InitContainerStatuses) + + nodeStore, _ := metricsRepo.AddNodeStore(pod.Spec.NodeName) + podId := NewPodId(pod.Namespace, pod.Name) + podStore, _ := nodeStore.AddPodStore(podId) + for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { - cuid := ContainerUID(pod.GetObjectMeta().GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) + metrics := NewContainerMetrics() - // Report container limits to PerfMetrics cache if cpu, ok := container.Resources.Limits["cpu"]; ok { if q, err := resource.ParseQuantity(cpu.String()); err == nil { - perfMetrics.ContainerCoresLimit.Set(cuid, float64(q.MilliValue())/1000) + metrics.CoresLimit = NewFloat64Metric(float64(q.MilliValue()) / 1000) } } if memory, ok := container.Resources.Limits["memory"]; ok { if q, err := resource.ParseQuantity(memory.String()); err == nil { - perfMetrics.ContainerMemLimit.Set(cuid, float64(q.Value())) + metrics.MemoryLimit = NewFloat64Metric(float64(q.Value())) } } + containerStore, _ := podStore.AddContainerStore(container.Name) + containerStore.SetContainerMetrics(metrics) + if s, ok := statuses[container.Name]; ok { // Extracting id and runtime ECS fields from ContainerID // which is in the form of :// @@ -249,6 +265,7 @@ func NewContainerMetadataEnricher( kubernetes2.ShouldPut(meta, "container.runtime", s.ContainerID[:split], base.Logger()) } } + id := join(pod.GetObjectMeta().GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) m[id] = meta } @@ -259,6 +276,10 @@ func NewContainerMetadataEnricher( if !ok { base.Logger().Debugf("Error while casting event: %s", ok) } + podId := NewPodId(pod.Namespace, pod.Name) + nodeStore := metricsRepo.GetNodeStore(pod.Spec.NodeName) + nodeStore.DeletePodStore(podId) + for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { id := join(pod.ObjectMeta.GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) delete(m, id) diff --git a/metricbeat/module/kubernetes/util/metrics_cache.go b/metricbeat/module/kubernetes/util/metrics_cache.go deleted file mode 100644 index 60bde73336e..00000000000 --- a/metricbeat/module/kubernetes/util/metrics_cache.go +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package util - -import ( - "time" - - "github.com/elastic/beats/v7/libbeat/common" -) - -// NewPerfMetricsCache initializes and returns a new PerfMetricsCache -func NewPerfMetricsCache(timeout time.Duration) *PerfMetricsCache { - return &PerfMetricsCache{ - NodeMemAllocatable: newValueMap(timeout), - NodeCoresAllocatable: newValueMap(timeout), - - ContainerMemLimit: newValueMap(timeout), - ContainerCoresLimit: newValueMap(timeout), - } -} - -// PerfMetricsCache stores known metrics from Kubernetes nodes and containers -type PerfMetricsCache struct { - NodeMemAllocatable *valueMap - NodeCoresAllocatable *valueMap - - ContainerMemLimit *valueMap - ContainerCoresLimit *valueMap -} - -// Start cache workers -func (c *PerfMetricsCache) Start() { - c.NodeMemAllocatable.Start() - c.NodeCoresAllocatable.Start() - c.ContainerMemLimit.Start() - c.ContainerCoresLimit.Start() -} - -// Stop cache workers -func (c *PerfMetricsCache) Stop() { - c.NodeMemAllocatable.Stop() - c.NodeCoresAllocatable.Stop() - c.ContainerMemLimit.Stop() - c.ContainerCoresLimit.Stop() -} - -// Returns the maximum timeout of all the caches under PerfMetricsCache -func (c *PerfMetricsCache) GetTimeout() time.Duration { - var ans time.Duration = 0 - - nmATimeout := c.NodeMemAllocatable.GetTimeout() - if nmATimeout > ans { - ans = nmATimeout - } - - ncATimeout := c.NodeCoresAllocatable.GetTimeout() - if ncATimeout > ans { - ans = ncATimeout - } - - cmLTimeout := c.ContainerMemLimit.GetTimeout() - if cmLTimeout > ans { - ans = cmLTimeout - } - - ccLTimeout := c.ContainerCoresLimit.GetTimeout() - if ccLTimeout > ans { - ans = ccLTimeout - } - return ans -} - -// Set the timeout of all the caches under PerfMetricsCache, then Stop and Start all the cache janitors -func (c *PerfMetricsCache) SetOrUpdateTimeout(timeout time.Duration) { - c.NodeMemAllocatable.SetTimeout(timeout) - c.NodeCoresAllocatable.SetTimeout(timeout) - c.ContainerMemLimit.SetTimeout(timeout) - c.ContainerCoresLimit.SetTimeout(timeout) - - c.Stop() - c.Start() -} - -type valueMap struct { - cache *common.Cache - timeout time.Duration -} - -func newValueMap(timeout time.Duration) *valueMap { - return &valueMap{ - cache: common.NewCache(timeout, 0), - timeout: timeout, - } -} - -// Get value -func (m *valueMap) Get(name string) float64 { - return m.GetWithDefault(name, 0.0) -} - -// Get value -func (m *valueMap) GetWithDefault(name string, def float64) float64 { - v := m.cache.Get(name) - if v, ok := v.(float64); ok { - return v - } - return def -} - -// Set value -func (m *valueMap) Set(name string, val float64) { - m.cache.PutWithTimeout(name, val, m.timeout) -} - -// Start cache workers -func (m *valueMap) Start() { - m.cache.StartJanitor(m.timeout) -} - -// Stop cache workers -func (m *valueMap) Stop() { - m.cache.StopJanitor() -} - -func (m *valueMap) GetTimeout() time.Duration { - return m.timeout -} - -func (m *valueMap) SetTimeout(timeout time.Duration) { - m.timeout = timeout -} - -// ContainerUID creates an unique ID for from namespace, pod name and container name -func ContainerUID(namespace, pod, container string) string { - return namespace + "/" + pod + "/" + container -} diff --git a/metricbeat/module/kubernetes/util/metrics_repo.go b/metricbeat/module/kubernetes/util/metrics_repo.go new file mode 100644 index 00000000000..c04328f2581 --- /dev/null +++ b/metricbeat/module/kubernetes/util/metrics_repo.go @@ -0,0 +1,334 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package util + +import ( + "strings" + "sync" +) + +// PodId defines a composite key for a Pod in NodeStore. A Pod is uniquely identified by a Namespace and a Name. +type PodId struct { + Namespace string + PodName string +} + +// Float64Metric is a wrapper for a float64 primitive type. The reason for this wrapper is to handle missing metrics with a `nil` pointer instead of a null value like `-1`. This is a better option since you could have metrics with negative values. +type Float64Metric struct { + Value float64 +} + +// ContainerMetrics contains all the metrics for a Container. +type ContainerMetrics struct { + CoresLimit *Float64Metric + MemoryLimit *Float64Metric +} + +// NodeMetrics contains all the metrics for a Node. +type NodeMetrics struct { + CoresAllocatable *Float64Metric + MemoryAllocatable *Float64Metric +} + +// ContainerStore contains the name of a container and its metrics. +type ContainerStore struct { + sync.RWMutex + ContainerName string + metrics *ContainerMetrics +} + +// PodStore contains the PodId of that Pod and a set of (containerName, ContainerStore) entries for each Container under a Pod. +type PodStore struct { + sync.RWMutex + PodId PodId + containers map[string]*ContainerStore +} + +// NodeStore contains the name of the node, the metrics for a Node and a set of (podId, PodStore) entries for each Pod under that Node. +type NodeStore struct { + sync.RWMutex + NodeName string + metrics *NodeMetrics + pods map[PodId]*PodStore +} + +// MetricsRepo contains a set of (nodeName, NodeStore) for each Node in the cluster. +type MetricsRepo struct { + sync.RWMutex + nodes map[string]*NodeStore +} + +// NewPodId returns a new PodId object given a Namespace and a Pod name. +func NewPodId(namespace, podName string) PodId { + return PodId{ + Namespace: namespace, + PodName: podName, + } +} + +// NewFloat64Metrics returns a Float64Metric given a float64 value. +func NewFloat64Metric(value float64) *Float64Metric { + return &Float64Metric{ + Value: value, + } +} + +// NewContainerMetrics creates an empty ContainerMetrics object. +func NewContainerMetrics() *ContainerMetrics { + return &ContainerMetrics{ + CoresLimit: nil, + MemoryLimit: nil, + } +} + +// NewNodeMetrics creates an empty NodeMetrics object. +func NewNodeMetrics() *NodeMetrics { + return &NodeMetrics{ + CoresAllocatable: nil, + MemoryAllocatable: nil, + } +} + +// NewContainerStore creates an empty ContainerStore object. +func NewContainerStore(containerName string) *ContainerStore { + ans := &ContainerStore{ + ContainerName: containerName, + metrics: NewContainerMetrics(), + } + return ans +} + +// NewPodStore creates an empty PodStore object. +func NewPodStore(podId PodId) *PodStore { + ans := &PodStore{ + PodId: podId, + containers: make(map[string]*ContainerStore), + } + return ans +} + +// NewNodeStore creates an empty NodeStore object. +func NewNodeStore(nodeName string) *NodeStore { + ans := &NodeStore{ + NodeName: nodeName, + metrics: NewNodeMetrics(), + pods: make(map[PodId]*PodStore), + } + return ans +} + +// NewMetricsRepo creates an empty MetricsRepo object. +func NewMetricsRepo() *MetricsRepo { + ans := &MetricsRepo{ + nodes: make(map[string]*NodeStore), + } + return ans +} + +// Clone clones a Float64Metric object. +func (m *Float64Metric) Clone() *Float64Metric { + return &Float64Metric{ + Value: m.Value, + } +} + +// Clone returns a copy of a ContainerMetrics object. +func (cm *ContainerMetrics) Clone() *ContainerMetrics { + ans := NewContainerMetrics() + if cm.CoresLimit != nil { + ans.CoresLimit = cm.CoresLimit.Clone() + } + if cm.MemoryLimit != nil { + ans.MemoryLimit = cm.MemoryLimit.Clone() + } + return ans +} + +// Clone returns a copy of a NodeMetric object. +func (nm *NodeMetrics) Clone() *NodeMetrics { + ans := NewNodeMetrics() + if nm.CoresAllocatable != nil { + ans.CoresAllocatable = nm.CoresAllocatable.Clone() + } + if nm.MemoryAllocatable != nil { + ans.MemoryAllocatable = nm.MemoryAllocatable.Clone() + } + return ans +} + +// DeleteNodeStore deletes a NodeStore from the MetricsRepo given the Node name. +func (mr *MetricsRepo) DeleteNodeStore(nodeName string) { + mr.Lock() + defer mr.Unlock() + delete(mr.nodes, nodeName) +} + +// DeleteAllNodeStore deletes all NodeStores from the MetricsRepo. +func (mr *MetricsRepo) DeleteAllNodeStore() { + mr.Lock() + defer mr.Unlock() + for nodeName := range mr.nodes { + delete(mr.nodes, nodeName) + } +} + +// NodeNames returns the names of all the Nodes. +func (mr *MetricsRepo) NodeNames() []string { + mr.RLock() + defer mr.RUnlock() + ans := make([]string, 0, len(mr.nodes)) + for nodeName := range mr.nodes { + ans = append(ans, nodeName) + } + return ans +} + +// PodIds returns the names of all the Pods under a Node. +func (ns *NodeStore) PodIds() []PodId { + ns.RLock() + defer ns.RUnlock() + ans := make([]PodId, 0, len(ns.pods)) + for podId := range ns.pods { + ans = append(ans, podId) + } + return ans +} + +// ContainerNames returns the names of all the Containers under a Pod. +func (ps *PodStore) ContainerNames() []string { + ps.RLock() + defer ps.RUnlock() + ans := make([]string, 0, len(ps.containers)) + for containerName := range ps.containers { + ans = append(ans, containerName) + } + return ans +} + +// AddNodeStore returns/create a NodeStore given a Node name. If the NodeStore already exists, it returns the object and `false` to indicate that it didn't create a new NodeStore. Otherwise if the NodeStore doesn't exists, it creates it and it returns the new object together with `true` to indicate that it created a new NodeStore. +func (mr *MetricsRepo) AddNodeStore(nodeName string) (*NodeStore, bool) { + mr.Lock() + defer mr.Unlock() + node, exists := mr.nodes[nodeName] + if !exists { + mr.nodes[nodeName] = NewNodeStore(nodeName) + return mr.nodes[nodeName], true + } + return node, false +} + +// GetNodeStore returns/create a NodeStore given a Node name. If the NodeStore already exists, it returns the object. Otherwise if the NodeStore doesn't exists, it creates an empty NodeStore and it returns it. This last behavior is to implement a [Null Object Design Pattern](https://en.wikipedia.org/wiki/Null_object_pattern). +func (mr *MetricsRepo) GetNodeStore(nodeName string) *NodeStore { + mr.RLock() + defer mr.RUnlock() + ans, exists := mr.nodes[nodeName] + if !exists { + return NewNodeStore(nodeName) + } + return ans +} + +// AddPodStore returns/create a PodStore given a PodId. If the PodStore already exists, it returns the object and `false` to indicate that it didn't create a new PodStore. Otherwise if the PodStore doesn't exists, it creates it and it returns the new object together with `true` to indicate that it created a new PodStore. +func (ns *NodeStore) AddPodStore(podId PodId) (*PodStore, bool) { + ns.Lock() + defer ns.Unlock() + pod, exists := ns.pods[podId] + if !exists { + ns.pods[podId] = NewPodStore(podId) + return ns.pods[podId], true + } + return pod, false +} + +// GetPodStore returns/create a PodStore given a PodId. If the PodStore already exists, it returns the object. Otherwise if the PodStore doesn't exists, it creates an empty PodStore and it returns it. This last behavior is to implement a [Null Object Design Pattern](https://en.wikipedia.org/wiki/Null_object_pattern). +func (ns *NodeStore) GetPodStore(podId PodId) *PodStore { + ns.RLock() + defer ns.RUnlock() + pod, exists := ns.pods[podId] + if !exists { + return NewPodStore(podId) + } + return pod +} + +// DeletePodStore delete a PodStore given a PodId from a NodeStore. +func (ns *NodeStore) DeletePodStore(podId PodId) { + ns.Lock() + defer ns.Unlock() + _, exists := ns.pods[podId] + if exists { + delete(ns.pods, podId) + } +} + +// GetNodeMetrics returns a copy of the Node metrics. +func (ns *NodeStore) GetNodeMetrics() *NodeMetrics { + ns.RLock() + defer ns.RUnlock() + return ns.metrics.Clone() +} + +// SetNodeMetrics set the Node metrics for a NodeStore. +func (ns *NodeStore) SetNodeMetrics(metrics *NodeMetrics) { + ns.Lock() + defer ns.Unlock() + ns.metrics = metrics +} + +// AddContainerStore returns/create a ContainerStore given a Container name. If the ContainerStore already exists, it returns the object and `false` to indicate that it didn't create a new ContainerStore. Otherwise if the ContainerStore doesn't exists, it creates it and it returns the new object together with `true` to indicate that it created a new ContainerStore. +func (ps *PodStore) AddContainerStore(containerName string) (*ContainerStore, bool) { + ps.Lock() + defer ps.Unlock() + container, exists := ps.containers[containerName] + if !exists { + ps.containers[containerName] = NewContainerStore(containerName) + return ps.containers[containerName], true + } + return container, false +} + +// GetContainerStore returns/create a ContainerStore given a Container name. If the ContainerStore already exists, it returns the object. Otherwise if the ContainerStore doesn't exists, it creates an empty ContainerStore and it returns it. This last behavior is to implement a [Null Object Design Pattern](https://en.wikipedia.org/wiki/Null_object_pattern). +func (ps *PodStore) GetContainerStore(containerName string) *ContainerStore { + ps.RLock() + defer ps.RUnlock() + container, exists := ps.containers[containerName] + if !exists { + return NewContainerStore(containerName) + } + return container +} + +// SetContainerMetrics set the container metrics. +func (cs *ContainerStore) SetContainerMetrics(metrics *ContainerMetrics) { + cs.Lock() + defer cs.Unlock() + cs.metrics = metrics +} + +// GetContainerMetrics returns a copy of the container metrics +func (cs *ContainerStore) GetContainerMetrics() *ContainerMetrics { + cs.RLock() + defer cs.RUnlock() + return cs.metrics.Clone() +} + +// String concatenates Namespace and PodName by "/" +func (pi PodId) String() string { + fields := []string{pi.Namespace, pi.PodName} + return strings.Join(fields, "/") +} diff --git a/metricbeat/module/kubernetes/util/metrics_repo_test.go b/metricbeat/module/kubernetes/util/metrics_repo_test.go new file mode 100644 index 00000000000..3de7122bca4 --- /dev/null +++ b/metricbeat/module/kubernetes/util/metrics_repo_test.go @@ -0,0 +1,388 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package util + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type MetricsRepoTestSuite struct { + suite.Suite + NodeName string + AnotherNodeName string + PodId PodId + AnotherPodId PodId + ContainerName string + AnotherContainerName string + ContainerMetric *ContainerMetrics + AnotherContainerMetric *ContainerMetrics + NodeMetric *NodeMetrics + AnotherNodeMetric *NodeMetrics + MetricValue float64 + MetricsRepo *MetricsRepo +} + +func (s *MetricsRepoTestSuite) SetupTest() { + s.MetricsRepo = NewMetricsRepo() + + s.NodeName = "node" + s.AnotherNodeName = "anotherNode" + + s.PodId = NewPodId("namespace", "pod") + s.AnotherPodId = NewPodId("namespace", "pod2") + + s.ContainerName = "container" + s.AnotherContainerName = "container2" + + s.ContainerMetric = NewContainerMetrics() + s.ContainerMetric.CoresLimit = NewFloat64Metric(0.2) + + s.AnotherContainerMetric = NewContainerMetrics() + s.AnotherContainerMetric.CoresLimit = NewFloat64Metric(0.3) + s.AnotherContainerMetric.MemoryLimit = NewFloat64Metric(50) + + s.NodeMetric = NewNodeMetrics() + s.NodeMetric.CoresAllocatable = NewFloat64Metric(0.001) + + s.AnotherNodeMetric = NewNodeMetrics() + s.AnotherNodeMetric.CoresAllocatable = NewFloat64Metric(0.002) + s.AnotherNodeMetric.MemoryAllocatable = NewFloat64Metric(60) +} + +func (s *MetricsRepoTestSuite) TestCloneContainerMetrics() { + s.MetricsRepo.DeleteAllNodeStore() + + newContainerMetric := s.ContainerMetric.Clone() + s.Equal(s.ContainerMetric, newContainerMetric) + s.True(s.ContainerMetric != newContainerMetric) + + anotherNewContainerMetric := s.AnotherContainerMetric.Clone() + s.Equal(s.AnotherContainerMetric, anotherNewContainerMetric) + s.True(s.AnotherContainerMetric != anotherNewContainerMetric) +} + +func (s *MetricsRepoTestSuite) TestCloneNodeMetrics() { + s.MetricsRepo.DeleteAllNodeStore() + + newNodeMetric := s.NodeMetric.Clone() + s.Equal(s.NodeMetric, newNodeMetric) + s.True(s.NodeMetric != newNodeMetric) + + anotherNewNodeMetric := s.AnotherNodeMetric.Clone() + s.Equal(s.AnotherNodeMetric, anotherNewNodeMetric) + s.True(s.AnotherNodeMetric != anotherNewNodeMetric) +} + +func (s *MetricsRepoTestSuite) TestNodeNames() { + s.MetricsRepo.DeleteAllNodeStore() + + s.Equal(0, len(s.MetricsRepo.NodeNames())) + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.AnotherNodeName, s.PodId, s.ContainerName, s.ContainerMetric) + + nodeNames := s.MetricsRepo.NodeNames() + s.Equal(2, len(nodeNames)) + s.Contains(nodeNames, s.NodeName) + s.Contains(nodeNames, s.AnotherNodeName) +} + +func (s *MetricsRepoTestSuite) TestPodNames() { + s.MetricsRepo.DeleteAllNodeStore() + + s.Equal(0, len(s.MetricsRepo.NodeNames())) + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.AnotherNodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.AnotherNodeName, s.AnotherPodId, s.ContainerName, s.AnotherContainerMetric) + + nodeStore := s.MetricsRepo.GetNodeStore(s.NodeName) + podNames := nodeStore.PodIds() + s.Equal(1, len(podNames)) + s.Contains(podNames, s.PodId) + + anotherNodeStore := s.MetricsRepo.GetNodeStore(s.AnotherNodeName) + anotherPodNames := anotherNodeStore.PodIds() + s.Equal(2, len(anotherPodNames)) + s.Contains(anotherPodNames, s.PodId) + s.Contains(anotherPodNames, s.AnotherPodId) +} + +func (s *MetricsRepoTestSuite) TestContainerNames() { + s.MetricsRepo.DeleteAllNodeStore() + + s.Equal(0, len(s.MetricsRepo.NodeNames())) + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.AnotherContainerName, s.ContainerMetric) + + nodeNames := s.MetricsRepo.NodeNames() + s.Equal(1, len(nodeNames)) + + nodeStore := s.MetricsRepo.GetNodeStore(s.NodeName) + podStore := nodeStore.GetPodStore(s.PodId) + containerNames := podStore.ContainerNames() + s.Equal(2, len(containerNames)) + s.Contains(containerNames, s.ContainerName) + s.Contains(containerNames, s.AnotherContainerName) +} + +func (s *MetricsRepoTestSuite) TestAddNodeStore() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, created := s.MetricsRepo.AddNodeStore(s.NodeName) + s.True(created) + + sameNodeStore, created := s.MetricsRepo.AddNodeStore(s.NodeName) + s.False(created) + + s.Equal(nodeStore, sameNodeStore) + s.True(nodeStore == sameNodeStore) + + anotherNodeStore, created := s.MetricsRepo.AddNodeStore(s.AnotherNodeName) + s.True(created) + + s.NotEqual(nodeStore, anotherNodeStore) + s.True(nodeStore != anotherNodeStore) +} + +func (s *MetricsRepoTestSuite) TestGetNodeStore() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, created := s.MetricsRepo.AddNodeStore(s.NodeName) + s.True(created) + + sameNodeStore := s.MetricsRepo.GetNodeStore(s.NodeName) + s.Equal(nodeStore, sameNodeStore) + s.True(nodeStore == sameNodeStore) + + anotherNodeStore := s.MetricsRepo.GetNodeStore(s.AnotherNodeName) + s.NotEqual(nodeStore, anotherNodeStore) + s.True(nodeStore != anotherNodeStore) +} + +func (s *MetricsRepoTestSuite) TestDeleteNodeStore() { + s.MetricsRepo.DeleteAllNodeStore() + + _, created := s.MetricsRepo.AddNodeStore(s.NodeName) + s.True(created) + + anotherNodeStore, created := s.MetricsRepo.AddNodeStore(s.AnotherNodeName) + s.True(created) + + s.Equal(2, len(s.MetricsRepo.NodeNames())) + + s.MetricsRepo.DeleteNodeStore(s.NodeName) + + s.Equal(1, len(s.MetricsRepo.NodeNames())) + + sameAnotherNodeStore := s.MetricsRepo.GetNodeStore(s.AnotherNodeName) + s.Equal(anotherNodeStore, sameAnotherNodeStore) + s.True(anotherNodeStore == sameAnotherNodeStore) +} + +func (s *MetricsRepoTestSuite) TestAddPodStore() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + podStore, created := nodeStore.AddPodStore(s.PodId) + s.True(created) + + samePodStore, created := nodeStore.AddPodStore(s.PodId) + s.False(created) + + s.Equal(podStore, samePodStore) + s.True(podStore == samePodStore) + + anotherPodStore, created := nodeStore.AddPodStore(s.AnotherPodId) + s.True(created) + + s.NotEqual(podStore, anotherPodStore) + s.True(podStore != anotherPodStore) +} + +func (s *MetricsRepoTestSuite) TestGetPodStore() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + podStore, created := nodeStore.AddPodStore(s.PodId) + s.True(created) + + samePodStore := nodeStore.GetPodStore(s.PodId) + s.Equal(podStore, samePodStore) + s.True(podStore == samePodStore) + + anotherPodStore := nodeStore.GetPodStore(s.AnotherPodId) + s.NotEqual(podStore, anotherPodStore) + s.True(podStore != anotherPodStore) +} + +func (s *MetricsRepoTestSuite) TestDeletePodStore() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + _, created := nodeStore.AddPodStore(s.PodId) + s.True(created) + + anotherPodStore, created := nodeStore.AddPodStore(s.AnotherPodId) + s.True(created) + + s.Equal(2, len(nodeStore.PodIds())) + + nodeStore.DeletePodStore(s.PodId) + s.Equal(1, len(nodeStore.PodIds())) + + sameAnotherPodStore := nodeStore.GetPodStore(s.AnotherPodId) + s.Equal(anotherPodStore, sameAnotherPodStore) + s.True(anotherPodStore == sameAnotherPodStore) +} + +func (s *MetricsRepoTestSuite) TestAddContainerStore() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + podStore, _ := nodeStore.AddPodStore(s.PodId) + containerStore, created := podStore.AddContainerStore(s.ContainerName) + s.True(created) + + sameContainerStore, created := podStore.AddContainerStore(s.ContainerName) + s.False(created) + + s.Equal(containerStore, sameContainerStore) + s.True(containerStore == sameContainerStore) + + anotherContainerStore, created := podStore.AddContainerStore(s.AnotherContainerName) + s.True(created) + + s.NotEqual(containerStore, anotherContainerStore) + s.True(containerStore != anotherContainerStore) +} + +func (s *MetricsRepoTestSuite) TestGetContainerMetrics() { + s.MetricsRepo.DeleteAllNodeStore() + + nodeStore, _ := s.MetricsRepo.AddNodeStore(s.NodeName) + podStore, _ := nodeStore.AddPodStore(s.PodId) + containerStore, created := podStore.AddContainerStore(s.ContainerName) + s.True(created) + + sameContainerStore := podStore.GetContainerStore(s.ContainerName) + s.Equal(containerStore, sameContainerStore) + s.True(containerStore == sameContainerStore) + + anotherContainerStore := podStore.GetContainerStore(s.AnotherContainerName) + s.NotEqual(containerStore, anotherContainerStore) + s.True(containerStore != anotherContainerStore) +} + +func (s *MetricsRepoTestSuite) TestSetContainerMetrics() { + s.MetricsRepo.DeleteAllNodeStore() + + s.Equal(0, len(s.MetricsRepo.NodeNames())) + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + + s.Equal(1, len(s.MetricsRepo.NodeNames())) + + s.Equal(s.ContainerMetric, GetMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName)) +} + +func (s *MetricsRepoTestSuite) TestSetContainerMetricsOverwrite() { + s.MetricsRepo.DeleteAllNodeStore() + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + + s.Equal(1, len(s.MetricsRepo.NodeNames())) + s.Equal(s.ContainerMetric, GetMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName)) + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.AnotherContainerMetric) + + s.Equal(1, len(s.MetricsRepo.NodeNames())) + s.Equal(s.AnotherContainerMetric, GetMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName)) +} + +func (s *MetricsRepoTestSuite) TestSetContainerMetricsSamePod() { + s.MetricsRepo.DeleteAllNodeStore() + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.AnotherContainerName, s.AnotherContainerMetric) + + s.Equal(1, len(s.MetricsRepo.NodeNames())) + s.Equal(s.ContainerMetric, GetMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName)) + s.Equal(s.AnotherContainerMetric, GetMetric(s.MetricsRepo, s.NodeName, s.PodId, s.AnotherContainerName)) + + nodeStore := s.MetricsRepo.GetNodeStore(s.NodeName) + s.Equal(1, len(nodeStore.PodIds())) +} + +func (s *MetricsRepoTestSuite) TestSetContainerMetricsMultiplePods() { + s.MetricsRepo.DeleteAllNodeStore() + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.NodeName, s.AnotherPodId, s.ContainerName, s.AnotherContainerMetric) + + nodeStore := s.MetricsRepo.GetNodeStore(s.NodeName) + s.Equal(2, len(nodeStore.PodIds())) +} + +func (s *MetricsRepoTestSuite) TestSetContainerMetricsMultipleNodes() { + s.MetricsRepo.DeleteAllNodeStore() + + addContainerMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName, s.ContainerMetric) + addContainerMetric(s.MetricsRepo, s.AnotherNodeName, s.AnotherPodId, s.ContainerName, s.AnotherContainerMetric) + + s.Equal(2, len(s.MetricsRepo.NodeNames())) + s.Equal(s.ContainerMetric, GetMetric(s.MetricsRepo, s.NodeName, s.PodId, s.ContainerName)) + s.Equal(s.AnotherContainerMetric, GetMetric(s.MetricsRepo, s.AnotherNodeName, s.AnotherPodId, s.ContainerName)) + + nodeStore := s.MetricsRepo.GetNodeStore(s.NodeName) + s.Equal(1, len(nodeStore.PodIds())) + + anotherNodeStore := s.MetricsRepo.GetNodeStore(s.AnotherNodeName) + s.Equal(1, len(anotherNodeStore.PodIds())) +} + +func (s *MetricsRepoTestSuite) TestGetContainerMetricsNotFound() { + s.MetricsRepo.DeleteAllNodeStore() + + ans := GetMetric(s.MetricsRepo, s.NodeName, s.AnotherPodId, s.ContainerName) + + s.Equal(0, len(s.MetricsRepo.NodeNames())) + s.Nil(ans.CoresLimit) + s.Nil(ans.MemoryLimit) +} + +func TestMetricsRepoTestSuite(t *testing.T) { + suite.Run(t, new(MetricsRepoTestSuite)) +} + +func addContainerMetric(metricsRepo *MetricsRepo, nodeName string, podId PodId, containerName string, containerMetric *ContainerMetrics) { + nodeStore, _ := metricsRepo.AddNodeStore(nodeName) + podStore, _ := nodeStore.AddPodStore(podId) + containerStore, _ := podStore.AddContainerStore(containerName) + containerStore.SetContainerMetrics(containerMetric) +} + +func GetMetric(metricsRepo *MetricsRepo, nodeName string, podId PodId, containerName string) *ContainerMetrics { + nodeStore := metricsRepo.GetNodeStore(nodeName) + podStore := nodeStore.GetPodStore(podId) + containerStore := podStore.GetContainerStore(containerName) + return containerStore.metrics +} diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index c091cdafa76..d24a21f653a 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.12 +FROM golang:1.18.5 RUN \ apt-get update \ diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index b6786c52375..f277db08f3c 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,15 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C packetbeat check; - make -C packetbeat update; - make check-no-changes; - cd x-pack/packetbeat; - mage check; - mage update; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/packetbeat/_meta/sample_outputs/flow.json b/packetbeat/_meta/sample_outputs/flow.json index 7fbcfaaa468..4ec6763063a 100644 --- a/packetbeat/_meta/sample_outputs/flow.json +++ b/packetbeat/_meta/sample_outputs/flow.json @@ -37,7 +37,7 @@ "start": "2018-11-30T01:15:50.119Z", "pid": 3468 }, - "mac": "08:00:27:d5:9d:5a", + "mac": "08-00-27-D5-9D-5A", "ip": "10.0.2.15", "port": 23377, "packets": 12 @@ -45,7 +45,7 @@ "destination": { "packets": 16, "bytes": 5558, - "mac": "52:54:00:12:35:02", + "mac": "52-54-00-12-35-02", "ip": "130.211.38.145", "port": 443 }, diff --git a/packetbeat/beater/packetbeat.go b/packetbeat/beater/packetbeat.go index a7c6899aae9..a0dde1e28a6 100644 --- a/packetbeat/beater/packetbeat.go +++ b/packetbeat/beater/packetbeat.go @@ -53,16 +53,12 @@ type flags struct { dumpfile *string } -var cmdLineArgs flags - -func init() { - cmdLineArgs = flags{ - file: flag.String("I", "", "Read packet data from specified file"), - loop: flag.Int("l", 1, "Loop file. 0 - loop forever"), - oneAtAtime: flag.Bool("O", false, "Read packets one at a time (press Enter)"), - topSpeed: flag.Bool("t", false, "Read packets as fast as possible, without sleeping"), - dumpfile: flag.String("dump", "", "Write all captured packets to this libpcap file"), - } +var cmdLineArgs = flags{ + file: flag.String("I", "", "Read packet data from specified file"), + loop: flag.Int("l", 1, "Loop file. 0 - loop forever"), + oneAtAtime: flag.Bool("O", false, "Read packets one at a time (press Enter)"), + topSpeed: flag.Bool("t", false, "Read packets as fast as possible, without sleeping"), + dumpfile: flag.String("dump", "", "Write all captured packets to this libpcap file"), } func initialConfig() config.Config { @@ -84,6 +80,7 @@ type packetbeat struct { done chan struct{} } +// New returns a new Packetbeat beat.Beater. func New(b *beat.Beat, rawConfig *conf.C) (beat.Beater, error) { configurator := config.NewAgentConfig if !b.Manager.Enabled() { @@ -110,6 +107,10 @@ func New(b *beat.Beat, rawConfig *conf.C) (beat.Beater, error) { }, nil } +// Run starts the packetbeat network capture, decoding and event publication, sending +// events to b.Publisher. If b is mananaged, packetbeat is registered with the +// reload.Registry and handled by fleet. Otherwise it is run until cancelled or a +// fatal error. func (pb *packetbeat) Run(b *beat.Beat) error { defer func() { if service.ProfileEnabled() { @@ -125,6 +126,8 @@ func (pb *packetbeat) Run(b *beat.Beat) error { return pb.runManaged(b, pb.factory) } +// runStatic constructs a packetbeat runner and starts it, returning on cancellation +// or the first fatal error. func (pb *packetbeat) runStatic(b *beat.Beat, factory *processorFactory) error { runner, err := factory.Create(b.Publisher, pb.config) if err != nil { @@ -144,6 +147,8 @@ func (pb *packetbeat) runStatic(b *beat.Beat, factory *processorFactory) error { return nil } +// runManaged registers a packetbeat runner with the reload.Registry and starts +// the runner by starting the beat's manager. It returns on the first fatal error. func (pb *packetbeat) runManaged(b *beat.Beat, factory *processorFactory) error { runner := newReloader(management.DebugK, factory, b.Publisher) reload.Register.MustRegisterList("inputs", runner) diff --git a/packetbeat/beater/processor.go b/packetbeat/beater/processor.go index 86bc9c974df..21adac2eac6 100644 --- a/packetbeat/beater/processor.go +++ b/packetbeat/beater/processor.go @@ -24,8 +24,10 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" + "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher/pipeline" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/beats/v7/packetbeat/config" "github.com/elastic/beats/v7/packetbeat/flows" @@ -69,7 +71,7 @@ func (p *processor) Start() { err := p.sniffer.Run() if err != nil { - p.err <- fmt.Errorf("sniffer loop failed: %v", err) + p.err <- fmt.Errorf("sniffer loop failed: %w", err) return } p.err <- nil @@ -90,6 +92,7 @@ func (p *processor) Stop() { p.publisher.Stop() } +// processorFactory controls construction of modules runners. type processorFactory struct { name string err chan error @@ -106,6 +109,7 @@ func newProcessorFactory(name string, err chan error, beat *beat.Beat, configura } } +// Create returns a new module runner that publishes to the provided pipeline, configured from cfg. func (p *processorFactory) Create(pipeline beat.PipelineConnector, cfg *conf.C) (cfgfile.Runner, error) { config, err := p.configurator(cfg) if err != nil { @@ -140,13 +144,13 @@ func (p *processorFactory) Create(pipeline beat.PipelineConnector, cfg *conf.C) protocols := protos.NewProtocols() err = protocols.Init(false, publisher, watcher, config.Protocols, config.ProtocolsList) if err != nil { - return nil, fmt.Errorf("Initializing protocol analyzers failed: %v", err) + return nil, fmt.Errorf("failed to initialize protocol analyzers: %w", err) } flows, err := setupFlows(pipeline, watcher, config) if err != nil { return nil, err } - sniffer, err := setupSniffer(config, protocols, workerFactory(publisher, protocols, watcher, flows, config)) + sniffer, err := setupSniffer(config, protocols, sniffer.DecodersFor(publisher, protocols, watcher, flows, config)) if err != nil { return nil, err } @@ -154,6 +158,51 @@ func (p *processorFactory) Create(pipeline beat.PipelineConnector, cfg *conf.C) return newProcessor(config.ShutdownTimeout, publisher, flows, sniffer, p.err), nil } +// setupFlows returns a *flows.Flows that will publish to the provided pipeline, +// configured with cfg and process enrichment via the provided watcher. +func setupFlows(pipeline beat.Pipeline, watcher procs.ProcessesWatcher, cfg config.Config) (*flows.Flows, error) { + if !cfg.Flows.IsEnabled() { + return nil, nil + } + + processors, err := processors.New(cfg.Flows.Processors) + if err != nil { + return nil, err + } + + var meta mapstr.M + if cfg.Flows.Index != "" { + meta = mapstr.M{"raw_index": cfg.Flows.Index} + } + client, err := pipeline.ConnectWith(beat.ClientConfig{ + Processing: beat.ProcessingConfig{ + EventMetadata: cfg.Flows.EventMetadata, + Processor: processors, + KeepNull: cfg.Flows.KeepNull, + Meta: meta, + }, + }) + if err != nil { + return nil, err + } + + return flows.NewFlows(client.PublishAll, watcher, cfg.Flows) +} + +func setupSniffer(cfg config.Config, protocols *protos.ProtocolsStruct, decoders sniffer.Decoders) (*sniffer.Sniffer, error) { + icmp, err := cfg.ICMP() + if err != nil { + return nil, err + } + + filter := cfg.Interfaces.BpfFilter + if filter == "" && !cfg.Flows.IsEnabled() { + filter = protocols.BpfFilter(cfg.Interfaces.WithVlans, icmp.Enabled()) + } + + return sniffer.New(false, filter, decoders, cfg.Interfaces) +} + // CheckConfig performs a dry-run creation of a Packetbeat pipeline based // on the provided configuration. This will involve setting up some dummy // sniffers and so will need libpcap to be loaded. diff --git a/packetbeat/beater/setup.go b/packetbeat/beater/setup.go deleted file mode 100644 index bc4a450a6cc..00000000000 --- a/packetbeat/beater/setup.go +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package beater - -import ( - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/processors" - "github.com/elastic/beats/v7/packetbeat/config" - "github.com/elastic/beats/v7/packetbeat/flows" - "github.com/elastic/beats/v7/packetbeat/procs" - "github.com/elastic/beats/v7/packetbeat/protos" - "github.com/elastic/beats/v7/packetbeat/sniffer" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func setupSniffer(cfg config.Config, protocols *protos.ProtocolsStruct, workerFactory sniffer.WorkerFactory) (*sniffer.Sniffer, error) { - icmp, err := cfg.ICMP() - if err != nil { - return nil, err - } - - filter := cfg.Interfaces.BpfFilter - if filter == "" && !cfg.Flows.IsEnabled() { - filter = protocols.BpfFilter(cfg.Interfaces.WithVlans, icmp.Enabled()) - } - - return sniffer.New(false, filter, workerFactory, cfg.Interfaces) -} - -func setupFlows(pipeline beat.Pipeline, watcher procs.ProcessesWatcher, cfg config.Config) (*flows.Flows, error) { - if !cfg.Flows.IsEnabled() { - return nil, nil - } - - processors, err := processors.New(cfg.Flows.Processors) - if err != nil { - return nil, err - } - - clientConfig := beat.ClientConfig{ - Processing: beat.ProcessingConfig{ - EventMetadata: cfg.Flows.EventMetadata, - Processor: processors, - KeepNull: cfg.Flows.KeepNull, - }, - } - if cfg.Flows.Index != "" { - clientConfig.Processing.Meta = mapstr.M{"raw_index": cfg.Flows.Index} - } - - client, err := pipeline.ConnectWith(clientConfig) - if err != nil { - return nil, err - } - - return flows.NewFlows(client.PublishAll, watcher, cfg.Flows) -} diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index 2ac1286b0dc..e12706a545f 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -340,7 +340,7 @@ Here’s an example of a flow information sent by Packetbeat. See "destination": { "bytes": 460, "ip": "198.51.100.2", - "mac": "06:05:04:03:02:01", + "mac": "06-05-04-03-02-01", "packets": 2, "port": 80 }, @@ -365,7 +365,7 @@ Here’s an example of a flow information sent by Packetbeat. See "source": { "bytes": 10, "ip": "203.0.113.3", - "mac": "01:02:03:04:05:06", + "mac": "01-02-03-04-05-06", "packets": 1, "port": 38901 } diff --git a/packetbeat/flows/flowid.go b/packetbeat/flows/flowid.go index eb2b7caec37..56c107df918 100644 --- a/packetbeat/flows/flowid.go +++ b/packetbeat/flows/flowid.go @@ -24,6 +24,7 @@ import ( "net" ) +// FlowID records flow details and statistics. type FlowID struct { rawFlowID flow Flow // remember associated flow for faster lookup @@ -35,6 +36,8 @@ type rawFlowID struct { dir flowDirection } +// flowIDMeta holds meta data describing the contents and layout +// of a rawFlowID's flowID buffer. type flowIDMeta struct { flags FlowIDFlag @@ -97,6 +100,9 @@ const ( SizeTCPFlowID + SizeUDPFlowID + SizeConnectionID + + // Ensure that SizeFlowIDMax is not greater than 255. + _ = uint8(SizeFlowIDMax) ) const offUnset uint8 = 0xff @@ -130,12 +136,6 @@ const ( flowDirReversed ) -func init() { - if SizeFlowIDMax > 255 { - panic("SizeFlowIDMax exceeds size limit") - } -} - func newFlowID() *FlowID { f := &FlowID{} f.init() @@ -240,12 +240,7 @@ func (f *FlowID) AddConnectionID(id uint64) { f.addID(&f.offID, ConnectionID, tmp[:], nil, flowDirUnset) } -func (f *FlowID) addMultLayerID( - off, outerOff *uint8, - flag, outerFlag FlowIDFlag, - a, b []byte, - hint flowDirection, -) { +func (f *FlowID) addMultLayerID(off, outerOff *uint8, flag, outerFlag FlowIDFlag, a, b []byte, hint flowDirection) { a, b = f.sortAddrWrite(a, b, hint) flags := f.flags & (flag | outerFlag) @@ -269,12 +264,7 @@ func (f *FlowID) addMultLayerID( } } -func (f *FlowID) addID( - off *uint8, - flag FlowIDFlag, - a, b []byte, - hint flowDirection, -) { +func (f *FlowID) addID(off *uint8, flag FlowIDFlag, a, b []byte, hint flowDirection) { a, b = f.sortAddrWrite(a, b, hint) if *off == offUnset { @@ -287,11 +277,7 @@ func (f *FlowID) addID( } } -func (f *FlowID) addWithPorts( - off *uint8, - flag FlowIDFlag, - src, dst uint16, -) { +func (f *FlowID) addWithPorts(off *uint8, flag FlowIDFlag, src, dst uint16) { var a, b [2]byte binary.LittleEndian.PutUint16(a[:], src) binary.LittleEndian.PutUint16(b[:], dst) @@ -372,9 +358,9 @@ func (f *rawFlowID) Get(i FlowIDFlag) []byte { func (f *rawFlowID) Serialize() []byte { buf := bytes.NewBuffer(nil) enc := base64.NewEncoder(base64.RawStdEncoding, buf) - + //nolint:errcheck // bytes.Buffer never returns a non-nil error on Write. enc.Write([]byte{ - byte(f.flags & 0xff), + byte(f.flags), byte(f.flags >> 8), f.offEth, f.offOutterVlan, @@ -392,6 +378,7 @@ func (f *rawFlowID) Serialize() []byte { f.cntVlan, f.cntIP, }) + //nolint:errcheck // bytes.Buffer never returns a non-nil error on Write. enc.Write(f.flowID) enc.Close() @@ -478,12 +465,7 @@ func (f *rawFlowID) extractID(off, sz uint8) []byte { if off == offUnset { return nil } - - { - off := int(off) - sz := int(sz) - return f.flowID[off : off+sz] - } + return f.flowID[off : int(off)+int(sz)] } func (f *rawFlowID) sortAddrRead(off, sz uint8) ([]byte, []byte, bool) { diff --git a/packetbeat/flows/flows.go b/packetbeat/flows/flows.go index 34aca4d104d..9abfccb51e3 100644 --- a/packetbeat/flows/flows.go +++ b/packetbeat/flows/flows.go @@ -26,12 +26,6 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type Flows struct { - worker *worker - table *flowMetaTable - counterReg *counterReg -} - // Reporter callback type, to report flow events to. type Reporter func([]beat.Event) @@ -42,6 +36,15 @@ const ( defaultPeriod = 10 * time.Second ) +// Flows holds and publishes network flow information for running processes. +type Flows struct { + worker *worker + table *flowMetaTable + counterReg *counterReg +} + +// NewFlows returns a Flows publishing to pub after enrichment by the given +// process watcher. Publication timeout and period are specified by config. func NewFlows(pub Reporter, watcher procs.ProcessesWatcher, config *config.Flows) (*Flows, error) { duration := func(s string, d time.Duration) (time.Duration, error) { if s == "" { @@ -101,11 +104,11 @@ func (f *Flows) Get(id *FlowID) *Flow { } func (f *Flows) Start() { - f.worker.Start() + f.worker.start() } func (f *Flows) Stop() { - f.worker.Stop() + f.worker.stop() } func (f *Flows) NewInt(name string) (*Int, error) { diff --git a/packetbeat/flows/flows_test.go b/packetbeat/flows/flows_test.go index 5eb2fbddb07..187dab3ac62 100644 --- a/packetbeat/flows/flows_test.go +++ b/packetbeat/flows/flows_test.go @@ -43,7 +43,7 @@ func (f *flowsChan) PublishFlows(events []beat.Event) { } func TestFlowsCounting(t *testing.T) { - logp.TestingSetup() + _ = logp.TestingSetup() mac1 := []byte{1, 2, 3, 4, 5, 6} mac2 := []byte{6, 5, 4, 3, 2, 1} @@ -87,8 +87,8 @@ func TestFlowsCounting(t *testing.T) { t.Fatalf("Failed to create flow worker: %v", err) } - worker.Start() - defer worker.Stop() + worker.start() + defer worker.stop() idForward := newFlowID() addrForward := addAll( @@ -141,8 +141,8 @@ func TestFlowsCounting(t *testing.T) { network := event["network"].(mapstr.M) // validate generated event - assert.Equal(t, net.HardwareAddr(mac1).String(), source["mac"]) - assert.Equal(t, net.HardwareAddr(mac2).String(), dest["mac"]) + assert.Equal(t, formatHardwareAddr(net.HardwareAddr(mac1)), source["mac"]) + assert.Equal(t, formatHardwareAddr(net.HardwareAddr(mac2)), dest["mac"]) assert.Equal(t, net.IP(ip1).String(), source["ip"]) assert.Equal(t, net.IP(ip2).String(), dest["ip"]) assert.Equal(t, uint16(256), source["port"]) diff --git a/packetbeat/flows/table.go b/packetbeat/flows/table.go index b3078f21a51..4cce712a4c5 100644 --- a/packetbeat/flows/table.go +++ b/packetbeat/flows/table.go @@ -44,6 +44,8 @@ type flowMetaTable struct { // Shared flow table. type flowTable struct { mutex sync.Mutex + + // table maps from flowID to biFlow. table map[string]*biFlow // linked list used to delete flows while iterating diff --git a/packetbeat/flows/util.go b/packetbeat/flows/util.go deleted file mode 100644 index 65d0b083560..00000000000 --- a/packetbeat/flows/util.go +++ /dev/null @@ -1,167 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package flows - -import ( - "sync" - "time" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/elastic-agent-libs/logp" -) - -type worker struct { - wg sync.WaitGroup - done chan struct{} - run func(*worker) -} - -type spool struct { - pub Reporter - events []beat.Event -} - -func newWorker(fn func(w *worker)) *worker { - return &worker{ - done: make(chan struct{}), - run: fn, - } -} - -func (w *worker) Start() { - debugf("start flows worker") - w.wg.Add(1) - go func() { - defer w.finished() - w.run(w) - }() -} - -func (w *worker) Stop() { - debugf("stop flows worker") - close(w.done) - w.wg.Wait() - debugf("stopped flows worker") -} - -func (w *worker) finished() { - w.wg.Done() - logp.Info("flows worker loop stopped") -} - -func (w *worker) sleep(d time.Duration) bool { - select { - case <-w.done: - return false - case <-time.After(d): - return true - } -} - -func (w *worker) tick(t *time.Ticker) bool { - select { - case <-w.done: - return false - case <-t.C: - return true - } -} - -func (w *worker) periodically(tick time.Duration, fn func() error) { - defer debugf("stop periodic loop") - - ticker := time.NewTicker(tick) - for { - cont := w.tick(ticker) - if !cont { - return - } - - err := fn() - if err != nil { - return - } - } -} - -func (s *spool) init(pub Reporter, sz int) { - s.pub = pub - s.events = make([]beat.Event, 0, sz) -} - -func (s *spool) publish(event beat.Event) { - s.events = append(s.events, event) - if len(s.events) == cap(s.events) { - s.flush() - } -} - -func (s *spool) flush() { - if len(s.events) == 0 { - return - } - - s.pub(s.events) - s.events = make([]beat.Event, 0, cap(s.events)) -} - -func gcd(a, b int64) int64 { - if a < 0 || b < 0 { - return 0 - } - - switch { - case a == b: - return a - case a == 0: - return b - case b == 0: - return a - } - - shift := uint(0) - for (a&1) == 0 && (b&1) == 0 { - shift++ - a /= 2 - b /= 2 - } - - for (a & 1) == 0 { - a = a / 2 - } - - // a is always odd - for { - for (b & 1) == 0 { - b = b / 2 - } - - // both a and b are odd. guaranteed b >= a - if a > b { - a, b = b, a - } - b -= a - - if b == 0 { - break - } - } - - // restore common factors of 2 - return a << shift -} diff --git a/packetbeat/flows/worker.go b/packetbeat/flows/worker.go index 7ac9f7125ca..0029e7acc16 100644 --- a/packetbeat/flows/worker.go +++ b/packetbeat/flows/worker.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "net" + "sync" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -28,61 +29,135 @@ import ( "github.com/elastic/beats/v7/libbeat/common/flowhash" "github.com/elastic/beats/v7/packetbeat/procs" "github.com/elastic/beats/v7/packetbeat/protos/applayer" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) -type flowsProcessor struct { - spool spool - watcher procs.ProcessesWatcher - table *flowMetaTable - counters *counterReg - timeout time.Duration -} - var ( ErrInvalidTimeout = errors.New("timeout must be >= 1s") ErrInvalidPeriod = errors.New("report period must be -1 or >= 1s") ) -func newFlowsWorker( - pub Reporter, - watcher procs.ProcessesWatcher, - table *flowMetaTable, - counters *counterReg, - timeout, period time.Duration, -) (*worker, error) { - oneSecond := 1 * time.Second +// worker is a generic asynchronous function processor. +type worker struct { + wg sync.WaitGroup + done chan struct{} + run func(*worker) +} + +// newWorker returns a handle to a worker to run fn. +func newWorker(fn func(w *worker)) *worker { + return &worker{ + done: make(chan struct{}), + run: fn, + } +} + +// start starts execution of the worker function. +func (w *worker) start() { + debugf("start flows worker") + w.wg.Add(1) + go func() { + defer w.finished() + w.run(w) + }() +} + +// finished decrements the workers working function count. finished +// must be called the same number of times as start over the lifetime +// of the worker. +func (w *worker) finished() { + w.wg.Done() + logp.Info("flows worker loop stopped") +} + +// stop terminates the function and waits until processing is complete. +// stop may only be called once. +func (w *worker) stop() { + debugf("stop flows worker") + close(w.done) + w.wg.Wait() + debugf("stopped flows worker") +} + +// sleep will sleep for the provided duration unless the worker has been +// stopped. sleep returns whether the worker can continue processing. +func (w *worker) sleep(d time.Duration) bool { + select { + case <-w.done: + return false + case <-time.After(d): + return true + } +} + +// tick will sleep until the provided ticker fires unless the worker has been +// stopped. tick returns whether the worker can continue processing. +func (w *worker) tick(t *time.Ticker) bool { + select { + case <-w.done: + return false + case <-t.C: + return true + } +} + +// periodically will execute fn each tick duration until the worker has been +// stopped or fn returns a non-nil error. +func (w *worker) periodically(tick time.Duration, fn func() error) { + defer debugf("stop periodic loop") + + ticker := time.NewTicker(tick) + defer ticker.Stop() + for { + cont := w.tick(ticker) + if !cont { + return + } + + err := fn() + if err != nil { + return + } + } +} - if timeout < oneSecond { +// newFlowsWorker returns a worker with a flow lifetime specified by timeout and a +// reporting intervals specified by period. If period is less than or equal to zero +// reporting will be done at flow lifetime end. +// Flows are published via the pub Reporter after being enriched with process information +// by watcher. +func newFlowsWorker(pub Reporter, watcher procs.ProcessesWatcher, table *flowMetaTable, counters *counterReg, timeout, period time.Duration) (*worker, error) { + if timeout < time.Second { return nil, ErrInvalidTimeout } - if 0 < period && period < oneSecond { + if 0 < period && period < time.Second { return nil, ErrInvalidPeriod } - tickDuration := timeout + tick := timeout ticksTimeout := 1 ticksPeriod := -1 if period > 0 { - tickDuration = time.Duration(gcd(int64(timeout), int64(period))) - if tickDuration < oneSecond { - tickDuration = oneSecond + tick = gcd(timeout, period) + if tick < time.Second { + tick = time.Second } - ticksTimeout = int(timeout / tickDuration) + ticksTimeout = int(timeout / tick) if ticksTimeout == 0 { ticksTimeout = 1 } - ticksPeriod = int(period / tickDuration) + ticksPeriod = int(period / tick) if ticksPeriod == 0 { ticksPeriod = 1 } } debugf("new flows worker. timeout=%v, period=%v, tick=%v, ticksTO=%v, ticksP=%v", - timeout, period, tickDuration, ticksTimeout, ticksPeriod) + timeout, period, tick, ticksTimeout, ticksPeriod) defaultBatchSize := 1024 processor := &flowsProcessor{ @@ -93,20 +168,26 @@ func newFlowsWorker( } processor.spool.init(pub, defaultBatchSize) - return makeWorker(processor, tickDuration, ticksTimeout, ticksPeriod, 10) + return makeWorker(processor, tick, ticksTimeout, ticksPeriod, 10) } -func makeWorker( - processor *flowsProcessor, - tickDuration time.Duration, - ticksTimeout, ticksPeriod int, - align int64, -) (*worker, error) { +// gcd returns the greatest common divisor of a and b. +func gcd(a, b time.Duration) time.Duration { + for b != 0 { + a, b = b, a%b + } + return a +} + +// makeWorker returns a worker that runs processor.execute each tick. Each timeout'th tick, +// the worker will check flow timeouts and each period'th tick, the worker will report flow +// events to be published. +func makeWorker(processor *flowsProcessor, tick time.Duration, timeout, period int, align int64) (*worker, error) { return newWorker(func(w *worker) { defer processor.execute(w, false, true, true) if align > 0 { - // round time to nearest 10 seconds for alignment + // Wait until the current time rounded up to nearest align seconds. aligned := time.Unix(((time.Now().Unix()+(align-1))/align)*align, 0) waitStart := time.Until(aligned) debugf("worker wait start(%v): %v", aligned, waitStart) @@ -115,22 +196,22 @@ func makeWorker( } } - nTimeout := ticksTimeout - nPeriod := ticksPeriod - reportPeriodically := ticksPeriod > 0 + nTimeout := timeout + nPeriod := period + reportPeriodically := period > 0 debugf("start flows worker loop") - w.periodically(tickDuration, func() error { + w.periodically(tick, func() error { nTimeout-- nPeriod-- debugf("worker tick, nTimeout=%v, nPeriod=%v", nTimeout, nPeriod) handleTimeout := nTimeout == 0 - handleReports := reportPeriodically && nPeriod == 0 if handleTimeout { - nTimeout = ticksTimeout + nTimeout = timeout } + handleReports := reportPeriodically && nPeriod == 0 if nPeriod <= 0 { - nPeriod = ticksPeriod + nPeriod = period } processor.execute(w, handleTimeout, handleReports, false) @@ -139,6 +220,14 @@ func makeWorker( }), nil } +type flowsProcessor struct { + spool spool + watcher procs.ProcessesWatcher + table *flowMetaTable + counters *counterReg + timeout time.Duration +} + func (fw *flowsProcessor) execute(w *worker, checkTimeout, handleReports, lastReport bool) { if !checkTimeout && !handleReports { return @@ -191,25 +280,14 @@ func (fw *flowsProcessor) execute(w *worker, checkTimeout, handleReports, lastRe fw.spool.flush() } -func (fw *flowsProcessor) report( - w *worker, - ts time.Time, - flow *biFlow, - isOver bool, - intNames, uintNames, floatNames []string, -) { +func (fw *flowsProcessor) report(w *worker, ts time.Time, flow *biFlow, isOver bool, intNames, uintNames, floatNames []string) { event := createEvent(fw.watcher, ts, flow, isOver, intNames, uintNames, floatNames) debugf("add event: %v", event) fw.spool.publish(event) } -func createEvent( - watcher procs.ProcessesWatcher, - ts time.Time, f *biFlow, - isOver bool, - intNames, uintNames, floatNames []string, -) beat.Event { +func createEvent(watcher procs.ProcessesWatcher, ts time.Time, f *biFlow, isOver bool, intNames, uintNames, floatNames []string) beat.Event { timestamp := ts event := mapstr.M{ @@ -245,8 +323,8 @@ func createEvent( // add ethernet layer meta data if src, dst, ok := f.id.EthAddr(); ok { - source["mac"] = net.HardwareAddr(src).String() - dest["mac"] = net.HardwareAddr(dst).String() + source["mac"] = formatHardwareAddr(net.HardwareAddr(src)) + dest["mac"] = formatHardwareAddr(net.HardwareAddr(dst)) } // add vlan @@ -442,10 +520,20 @@ func createEvent( } } -func encodeStats( - stats *flowStats, - ints, uints, floats []string, -) map[string]interface{} { +// formatHardwareAddr formats hardware addresses according to the ECS spec. +func formatHardwareAddr(addr net.HardwareAddr) string { + buf := make([]byte, 0, len(addr)*3-1) + for _, b := range addr { + if len(buf) != 0 { + buf = append(buf, '-') + } + const hexDigit = "0123456789ABCDEF" + buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf]) + } + return string(buf) +} + +func encodeStats(stats *flowStats, ints, uints, floats []string) map[string]interface{} { report := make(map[string]interface{}) i := 0 @@ -514,9 +602,48 @@ func putOrAppendUint64(m mapstr.M, key string, value uint64) { case uint32: m[key] = []uint64{uint64(v), value} case uint64: - m[key] = []uint64{uint64(v), value} + m[key] = []uint64{v, value} case []uint64: m[key] = append(v, value) } } } + +// spool is an event publisher spool. +type spool struct { + pub Reporter + events []beat.Event +} + +// init sets the destination and spool size. +func (s *spool) init(pub Reporter, sz int) { + s.pub = pub + s.events = make([]beat.Event, 0, sz) +} + +// publish queues the event for publication, flushing to the destination +// if the spool is full. +func (s *spool) publish(event beat.Event) { + s.events = append(s.events, event) + if len(s.events) == cap(s.events) { + s.flush() + } +} + +// flush sends the spooled events to the destination and clears them +// from the spool. +func (s *spool) flush() { + if len(s.events) == 0 { + return + } + s.pub(s.events) + // A newly allocated spool is created since the + // elements of s.events are no longer owned by s + // during testing and mutating them causes a panic. + // + // The beat.Client interface which Reporter is + // derived from is silent on whether the caller + // is allowed to modify elements of the slice + // after the call to the PublishAll method returns. + s.events = make([]beat.Event, 0, cap(s.events)) +} diff --git a/packetbeat/flows/worker_test.go b/packetbeat/flows/worker_test.go index ad6706866cf..2b5b1261252 100644 --- a/packetbeat/flows/worker_test.go +++ b/packetbeat/flows/worker_test.go @@ -20,7 +20,7 @@ package flows import ( "encoding/json" "flag" - "io/ioutil" + "os" "testing" "time" @@ -37,7 +37,7 @@ import ( var dataFlag = flag.Bool("data", false, "Write updated data.json files") func TestCreateEvent(t *testing.T) { - logp.TestingSetup() + _ = logp.TestingSetup() // Build biflow event. start := time.Unix(1542292881, 0) @@ -70,14 +70,14 @@ func TestCreateEvent(t *testing.T) { // Validate the contents of the event. validate := lookslike.MustCompile(map[string]interface{}{ "source": map[string]interface{}{ - "mac": "01:02:03:04:05:06", + "mac": "01-02-03-04-05-06", "ip": "203.0.113.3", "port": port1, "bytes": uint64(10), "packets": uint64(1), }, "destination": map[string]interface{}{ - "mac": "06:05:04:03:02:01", + "mac": "06-05-04-03-02-01", "ip": "198.51.100.2", "port": port2, "bytes": uint64(460), @@ -116,13 +116,13 @@ func TestCreateEvent(t *testing.T) { // Write the event to disk if -data is used. if *dataFlag { - event.Fields.Put("@timestamp", common.Time(end)) + event.Fields.Put("@timestamp", common.Time(end)) //nolint:errcheck // Never fails. output, err := json.MarshalIndent(&event.Fields, "", " ") if err != nil { t.Fatal(err) } - if err := ioutil.WriteFile("../_meta/sample_outputs/flow.json", output, 0o644); err != nil { + if err := os.WriteFile("../_meta/sample_outputs/flow.json", output, 0o644); err != nil { t.Fatal(err) } } diff --git a/packetbeat/procs/procs.go b/packetbeat/procs/procs.go index 0ec0329551a..b8bf402d096 100644 --- a/packetbeat/procs/procs.go +++ b/packetbeat/procs/procs.go @@ -31,63 +31,73 @@ import ( // This controls how often process info for a running process is reloaded // A big value means less unnecessary refreshes at a higher risk of missing // a PID being recycled by the OS -const processCacheExpiration = time.Second * 30 +const processCacheExpiration = 30 * time.Second var ( anyIPv4 = net.IPv4zero.String() anyIPv6 = net.IPv6unspecified.String() ) +// ProcessWatcher implements process enrichment for network traffic. +type ProcessesWatcher struct { + portProcMap map[applayer.Transport]map[endpoint]portProcMapping + localAddrs []net.IP // localAddrs lists IP addresses that are to be treated as local. + processCache map[int]*process // processCache is a time-expiration cache of process details keyed on PID. + + enabled bool // enabled specifier whether the ProcessWatcher will be active. + monitored []ProcConfig // monitored is the set of processes that are monitored by the ProcessWatcher. + + // watcher is the OS-dependent engine for the ProcessWatcher. + watcher processWatcher +} + +// endpoint is a network address/port number complex. type endpoint struct { address string port uint16 } +// portProcMapping is an association between an endpoint and a process. type portProcMapping struct { - endpoint endpoint + endpoint endpoint // FIXME: This is never used. pid int proc *process } +// process describes an OS process. type process struct { pid, ppid int name, exe, cwd string args []string startTime time.Time - // To control cache expiration - expiration time.Time + // expires is the time at which the process will be dropped + // from the cache during enrichment queries. + expires time.Time +} + +// Init initializes the ProcessWatcher with the provided configuration. +func (proc *ProcessesWatcher) Init(config ProcsConfig) error { + return proc.init(config, proc) } -// Allow the OS-dependent implementation to be replaced by a mock for testing -type processWatcherImpl interface { +// processWatcher allows the OS-dependent implementation to be replaced by a mock for testing +type processWatcher interface { // GetLocalPortToPIDMapping returns the list of local port numbers and the PID // that owns them. GetLocalPortToPIDMapping(transport applayer.Transport) (ports map[endpoint]int, err error) + // GetProcess returns the process metadata. GetProcess(pid int) *process - // GetLocalIPs returns the list of local addresses. - GetLocalIPs() ([]net.IP, error) -} - -type ProcessesWatcher struct { - portProcMap map[applayer.Transport]map[endpoint]portProcMapping - localAddrs []net.IP - processCache map[int]*process - - // config - enabled bool - procConfig []ProcConfig - impl processWatcherImpl -} - -func (proc *ProcessesWatcher) Init(config ProcsConfig) error { - return proc.initWithImpl(config, proc) + // GetLocalIPs returns the list of local addresses. If the returned error + // is non-nil, the IP slice is nil. + GetLocalIPs() ([]net.IP, error) } -func (proc *ProcessesWatcher) initWithImpl(config ProcsConfig, impl processWatcherImpl) error { - proc.impl = impl +// init sets up the necessary data structures for the ProcessWatcher. +func (proc *ProcessesWatcher) init(config ProcsConfig, watcher processWatcher) error { + proc.watcher = watcher proc.portProcMap = map[applayer.Transport]map[endpoint]portProcMapping{ applayer.TransportUDP: make(map[endpoint]portProcMapping), applayer.TransportTCP: make(map[endpoint]portProcMapping), @@ -102,15 +112,14 @@ func (proc *ProcessesWatcher) initWithImpl(config ProcsConfig, impl processWatch logp.Info("Process watcher disabled") } - // Read the local IP addresses + // Read the local IP addresses. var err error - proc.localAddrs, err = impl.GetLocalIPs() + proc.localAddrs, err = watcher.GetLocalIPs() if err != nil { logp.Err("Error getting local IP addresses: %s", err) - proc.localAddrs = []net.IP{} } - proc.procConfig = config.Monitored + proc.monitored = config.Monitored return nil } @@ -139,6 +148,8 @@ func (proc *ProcessesWatcher) FindProcessesTuple(tuple *common.IPPortTuple, tran return &procTuple } +// enrich adds process information to dst for the process associated with the given IP, port and +// transport if the IP is not local and the information is available to the ProcessWatcher. func (proc *ProcessesWatcher) enrich(dst *common.Process, ip net.IP, port uint16, transport applayer.Transport) { if !proc.isLocalIP(ip) { return @@ -158,7 +169,21 @@ func (proc *ProcessesWatcher) enrich(dst *common.Process, ip net.IP, port uint16 } } +func (proc *ProcessesWatcher) isLocalIP(ip net.IP) bool { + if ip.IsLoopback() { + return true + } + for _, addr := range proc.localAddrs { + if ip.Equal(addr) { + return true + } + } + return false +} + func (proc *ProcessesWatcher) findProc(address net.IP, port uint16, transport applayer.Transport) *process { + // This should not be necessary; none of the + // dependency code panics in normal operation. defer logp.Recover("FindProc exception") procMap, ok := proc.portProcMap[transport] @@ -207,7 +232,7 @@ func (proc *ProcessesWatcher) updateMap(transport applayer.Transport) { }() } - endpoints, err := proc.impl.GetLocalPortToPIDMapping(transport) + endpoints, err := proc.watcher.GetLocalPortToPIDMapping(transport) if err != nil { logp.Err("unable to list local ports: %v", err) } @@ -219,6 +244,15 @@ func (proc *ProcessesWatcher) updateMap(transport applayer.Transport) { } } +func (proc *ProcessesWatcher) expireProcessCache() { + now := time.Now() + for pid, info := range proc.processCache { + if now.After(info.expires) { + delete(proc.processCache, pid) + } + } +} + func (proc *ProcessesWatcher) updateMappingEntry(transport applayer.Transport, e endpoint, pid int) { prev, ok := proc.portProcMap[transport][e] if ok && prev.pid == pid { @@ -243,33 +277,26 @@ func (proc *ProcessesWatcher) updateMappingEntry(transport applayer.Transport, e } } -func (proc *ProcessesWatcher) isLocalIP(ip net.IP) bool { - if ip.IsLoopback() { - return true - } - - for _, addr := range proc.localAddrs { - if ip.Equal(addr) { - return true - } - } - - return false -} - +// getProcessInfo returns a potentially cached process corresponding to the +// provided process ID. +// +// If any part of the process's argv contains a substring in proc.monitored.CmdlineGrep, +// the name of the process is replaced with the corresponding proc.monitored.Process. +// This behaviour is not recommended to be used and is not available to integrations +// packages by design. func (proc *ProcessesWatcher) getProcessInfo(pid int) *process { if p, ok := proc.processCache[pid]; ok { return p } // Not in cache, resolve process info - p := proc.impl.GetProcess(pid) + p := proc.watcher.GetProcess(pid) if p == nil { return nil } // The packetbeat.procs.monitored*.cmdline_grep allows you to overwrite // the process name with an alias. - for _, match := range proc.procConfig { + for _, match := range proc.monitored { if strings.Contains(strings.Join(p.args, " "), match.CmdlineGrep) { p.name = match.Process break @@ -279,15 +306,6 @@ func (proc *ProcessesWatcher) getProcessInfo(pid int) *process { return p } -func (proc *ProcessesWatcher) expireProcessCache() { - now := time.Now() - for pid, info := range proc.processCache { - if now.After(info.expiration) { - delete(proc.processCache, pid) - } - } -} - // GetProcess returns the process metadata. func (proc *ProcessesWatcher) GetProcess(pid int) *process { if pid <= 0 { @@ -307,14 +325,14 @@ func (proc *ProcessesWatcher) GetProcess(pid int) *process { } return &process{ - pid: info.PID, - ppid: info.PPID, - name: procName(info), - exe: info.Exe, - cwd: info.CWD, - args: info.Args, - startTime: info.StartTime, - expiration: time.Now().Add(processCacheExpiration), + pid: info.PID, + ppid: info.PPID, + name: procName(info), + exe: info.Exe, + cwd: info.CWD, + args: info.Args, + startTime: info.StartTime, + expires: time.Now().Add(processCacheExpiration), } } diff --git a/packetbeat/procs/procs_linux.go b/packetbeat/procs/procs_linux.go index 9034ef9c26f..778a3a00551 100644 --- a/packetbeat/procs/procs_linux.go +++ b/packetbeat/procs/procs_linux.go @@ -53,21 +53,6 @@ func procName(info types.ProcessInfo) string { return filepath.Base(info.Args[0]) } -type socketInfo struct { - srcIP, dstIP net.IP - srcPort, dstPort uint16 - - uid uint32 - inode uint64 -} - -var procFiles = map[applayer.Transport]struct { - ipv4, ipv6 string -}{ - applayer.TransportUDP: {"/proc/net/udp", "/proc/net/udp6"}, - applayer.TransportTCP: {"/proc/net/tcp", "/proc/net/tcp6"}, -} - var warnIPv6Once sync.Once // GetLocalPortToPIDMapping returns the list of local port numbers and the PID @@ -126,6 +111,13 @@ func (proc *ProcessesWatcher) GetLocalPortToPIDMapping(transport applayer.Transp return ports, nil } +var procFiles = map[applayer.Transport]struct { + ipv4, ipv6 string +}{ + applayer.TransportUDP: {"/proc/net/udp", "/proc/net/udp6"}, + applayer.TransportTCP: {"/proc/net/tcp", "/proc/net/tcp6"}, +} + func findSocketsOfPid(prefix string, pid int) (inodes []uint64, err error) { dirname := filepath.Join(prefix, "/proc", strconv.Itoa(pid), "fd") procfs, err := os.Open(dirname) @@ -159,8 +151,19 @@ func findSocketsOfPid(prefix string, pid int) (inodes []uint64, err error) { return inodes, nil } -func socketsFromProc(filename string, ipv6 bool) ([]*socketInfo, error) { - file, err := os.Open(filename) +// socketInfo hold details for network sockets obtained from /proc/net. +type socketInfo struct { + srcIP, dstIP net.IP + srcPort, dstPort uint16 + + uid uint32 // uid is the effective UID of the process that created the socket. + inode uint64 // inode is the inode of the file corresponding to the socket. +} + +// socketsFromProc returns the socket information held in the the /proc/net file +// at path. +func socketsFromProc(path string, ipv6 bool) ([]*socketInfo, error) { + file, err := os.Open(path) if err != nil { return nil, err } diff --git a/packetbeat/procs/procs_linux_test.go b/packetbeat/procs/procs_linux_test.go index 62b3696923d..5a90af037d7 100644 --- a/packetbeat/procs/procs_linux_test.go +++ b/packetbeat/procs/procs_linux_test.go @@ -29,74 +29,8 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type testProcFile struct { - path string - contents string - isLink bool -} - -func createFakeDirectoryStructure(prefix string, files []testProcFile) error { - var err error - for _, file := range files { - dir := filepath.Dir(file.path) - err = os.MkdirAll(filepath.Join(prefix, dir), 0o755) - if err != nil { - return err - } - - if !file.isLink { - err = ioutil.WriteFile(filepath.Join(prefix, file.path), - []byte(file.contents), 0o644) - if err != nil { - return err - } - } else { - err = os.Symlink(file.contents, filepath.Join(prefix, file.path)) - if err != nil { - return err - } - } - } - - return nil -} - -func assertIntArraysAreEqual(t *testing.T, expected []int, result []int) bool { - for _, ex := range expected { - found := false - for _, res := range result { - if ex == res { - found = true - break - } - } - if !found { - t.Errorf("Expected array %v but got %v", expected, result) - return false - } - } - return true -} - -func assertUint64ArraysAreEqual(t *testing.T, expected []uint64, result []uint64) bool { - for _, ex := range expected { - found := false - for _, res := range result { - if ex == res { - found = true - break - } - } - if !found { - t.Errorf("Expected array %v but got %v", expected, result) - return false - } - } - return true -} - func TestFindSocketsOfPid(t *testing.T) { - logp.TestingSetup() + _ = logp.TestingSetup() proc := []testProcFile{ {path: "/proc/766/fd/0", isLink: true, contents: "/dev/null"}, @@ -166,3 +100,52 @@ func TestParse_Proc_Net_Tcp6(t *testing.T) { t.Error("Failed to parse source IP address 2001:db8::123:ffff:89ab:cdef, got instead", socketInfo[4].srcIP.String()) } } + +type testProcFile struct { + path string + contents string + isLink bool +} + +func createFakeDirectoryStructure(prefix string, files []testProcFile) error { + var err error + for _, file := range files { + dir := filepath.Dir(file.path) + err = os.MkdirAll(filepath.Join(prefix, dir), 0o755) + if err != nil { + return err + } + + if !file.isLink { + err = ioutil.WriteFile(filepath.Join(prefix, file.path), + []byte(file.contents), 0o644) + if err != nil { + return err + } + } else { + err = os.Symlink(file.contents, filepath.Join(prefix, file.path)) + if err != nil { + return err + } + } + } + + return nil +} + +func assertUint64ArraysAreEqual(t *testing.T, expected []uint64, result []uint64) bool { + for _, ex := range expected { + found := false + for _, res := range result { + if ex == res { + found = true + break + } + } + if !found { + t.Errorf("Expected array %v but got %v", expected, result) + return false + } + } + return true +} diff --git a/packetbeat/procs/procs_test.go b/packetbeat/procs/procs_test.go index 6a8a6380c23..19f2171d8b8 100644 --- a/packetbeat/procs/procs_test.go +++ b/packetbeat/procs/procs_test.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type testingImpl struct { +type mockWatcher struct { localIPs []net.IP portToPID map[applayer.Transport]map[endpoint]int pidToProcess map[int]*process @@ -44,8 +44,8 @@ type runningProcess struct { proto applayer.Transport } -func newTestingImpl(localIPs []net.IP, processes []runningProcess) *testingImpl { - impl := &testingImpl{ +func newMockWatcher(localIPs []net.IP, processes []runningProcess) *mockWatcher { + w := &mockWatcher{ localIPs: localIPs, portToPID: map[applayer.Transport]map[endpoint]int{ applayer.TransportTCP: make(map[endpoint]int), @@ -55,31 +55,31 @@ func newTestingImpl(localIPs []net.IP, processes []runningProcess) *testingImpl } for i, proc := range processes { for _, port := range proc.ports { - impl.portToPID[proc.proto][port] = proc.pid + w.portToPID[proc.proto][port] = proc.pid } - impl.pidToProcess[proc.pid] = &processes[i].process + w.pidToProcess[proc.pid] = &processes[i].process } - return impl + return w } -func (impl *testingImpl) GetLocalPortToPIDMapping(transport applayer.Transport) (ports map[endpoint]int, err error) { - return impl.portToPID[transport], nil +func (w *mockWatcher) GetLocalPortToPIDMapping(transport applayer.Transport) (ports map[endpoint]int, err error) { + return w.portToPID[transport], nil } -func (impl *testingImpl) GetProcess(pid int) *process { - if cmdline, ok := impl.pidToProcess[pid]; ok { +func (w *mockWatcher) GetProcess(pid int) *process { + if cmdline, ok := w.pidToProcess[pid]; ok { return cmdline } return nil } -func (impl *testingImpl) GetLocalIPs() ([]net.IP, error) { - return impl.localIPs, nil +func (w *mockWatcher) GetLocalIPs() ([]net.IP, error) { + return w.localIPs, nil } func TestFindProcessTuple(t *testing.T) { - logp.TestingSetup() + _ = logp.TestingSetup() config := ProcsConfig{ Enabled: true, Monitored: []ProcConfig{ @@ -88,7 +88,7 @@ func TestFindProcessTuple(t *testing.T) { {Process: "NMap", CmdlineGrep: "nmap"}, }, } - impl := newTestingImpl( + w := newMockWatcher( []net.IP{ net.ParseIP("127.0.0.1"), net.ParseIP("192.168.1.1"), @@ -177,7 +177,7 @@ func TestFindProcessTuple(t *testing.T) { }, }) procs := ProcessesWatcher{} - err := procs.initWithImpl(config, impl) + err := procs.init(config, w) assert.NoError(t, err) for _, testCase := range []struct { @@ -240,8 +240,8 @@ func TestFindProcessTuple(t *testing.T) { name: "New client", preAction: func() { // add a new running process - impl.pidToProcess[555] = &process{args: strings.Fields("/usr/bin/nmap -sT -P443 10.0.0.0/8")} - impl.portToPID[applayer.TransportTCP][endpoint{anyIPv6, 55555}] = 555 + w.pidToProcess[555] = &process{args: strings.Fields("/usr/bin/nmap -sT -P443 10.0.0.0/8")} + w.portToPID[applayer.TransportTCP][endpoint{anyIPv6, 55555}] = 555 }, proto: applayer.TransportTCP, srcIP: "7777::33", srcPort: 55555, diff --git a/packetbeat/procs/procs_windows.go b/packetbeat/procs/procs_windows.go index a011fdcbfa1..a03e0b95a7b 100644 --- a/packetbeat/procs/procs_windows.go +++ b/packetbeat/procs/procs_windows.go @@ -21,9 +21,9 @@ package procs import ( - "encoding/binary" "errors" "fmt" + "math/bits" "net" "syscall" "unsafe" @@ -34,43 +34,6 @@ import ( "github.com/elastic/go-sysinfo/types" ) -var machineEndiannes = getMachineEndiannes() - -type extractor interface { - // Extract extracts useful information from the pointed-to structure - Extract(unsafe.Pointer) - // Size of the structure - Size() int -} - -type ( - callbackFn func(net.IP, uint16, int) - extractorFactory func(fn callbackFn) extractor -) - -type ( - tcpRowOwnerPIDExtractor callbackFn - tcp6RowOwnerPIDExtractor callbackFn - udpRowOwnerPIDExtractor callbackFn - udp6RowOwnerPIDExtractor callbackFn -) - -var tablesByTransport = map[applayer.Transport][]struct { - family uint32 - function GetExtendedTableFn - class uint32 - extractor extractorFactory -}{ - applayer.TransportTCP: { - {windows.AF_INET, _GetExtendedTcpTable, TCP_TABLE_OWNER_PID_ALL, extractTCPRowOwnerPID}, - {windows.AF_INET6, _GetExtendedTcpTable, TCP_TABLE_OWNER_PID_ALL, extractTCP6RowOwnerPID}, - }, - applayer.TransportUDP: { - {windows.AF_INET, _GetExtendedUdpTable, UDP_TABLE_OWNER_PID, extractUDPRowOwnerPID}, - {windows.AF_INET6, _GetExtendedUdpTable, UDP_TABLE_OWNER_PID, extractUDP6RowOwnerPID}, - }, -} - // procName returns the name for the process. func procName(info types.ProcessInfo) string { return info.Name @@ -102,6 +65,31 @@ func (proc *ProcessesWatcher) GetLocalPortToPIDMapping(transport applayer.Transp return ports, nil } +type extractor interface { + // Extract extracts useful information from the pointed-to structure + Extract(unsafe.Pointer) + // Size of the structure + Size() int +} + +type callbackFn func(net.IP, uint16, int) + +var tablesByTransport = map[applayer.Transport][]struct { + family uint32 + function GetExtendedTableFn + class uint32 + extractor func(fn callbackFn) extractor +}{ + applayer.TransportTCP: { + {windows.AF_INET, _GetExtendedTcpTable, TCP_TABLE_OWNER_PID_ALL, extractTCPRowOwnerPID}, + {windows.AF_INET6, _GetExtendedTcpTable, TCP_TABLE_OWNER_PID_ALL, extractTCP6RowOwnerPID}, + }, + applayer.TransportUDP: { + {windows.AF_INET, _GetExtendedUdpTable, UDP_TABLE_OWNER_PID, extractUDPRowOwnerPID}, + {windows.AF_INET6, _GetExtendedUdpTable, UDP_TABLE_OWNER_PID, extractUDP6RowOwnerPID}, + }, +} + func getNetTable(fn GetExtendedTableFn, order bool, family uint32, tableClass uint32) ([]byte, error) { // Call the winapi function with an increasing buffer until the required // size is satisfied @@ -135,13 +123,6 @@ func parseTable(data []byte, extractor extractor) error { return nil } -// The MIB_TCP_ROW_xxx structures uses a 32-bit field to store ports: -// The first 16 bits contain the port in big-endian encoding -// The last 16 bits are unused. -func uint32FieldToPort(be uint32) uint16 { - return binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&be))[:]) -} - func extractTCPRowOwnerPID(fn callbackFn) extractor { return tcpRowOwnerPIDExtractor(fn) } @@ -158,6 +139,8 @@ func extractUDP6RowOwnerPID(fn callbackFn) extractor { return udp6RowOwnerPIDExtractor(fn) } +type tcpRowOwnerPIDExtractor callbackFn + // Extract will parse a row of Size() bytes pointed to by ptr func (e tcpRowOwnerPIDExtractor) Extract(ptr unsafe.Pointer) { row := (*TCPRowOwnerPID)(ptr) @@ -169,6 +152,8 @@ func (tcpRowOwnerPIDExtractor) Size() int { return int(unsafe.Sizeof(TCPRowOwnerPID{})) } +type tcp6RowOwnerPIDExtractor callbackFn + // Extract will parse a row of Size() bytes pointed to by ptr func (e tcp6RowOwnerPIDExtractor) Extract(ptr unsafe.Pointer) { row := (*TCP6RowOwnerPID)(ptr) @@ -180,6 +165,8 @@ func (tcp6RowOwnerPIDExtractor) Size() int { return int(unsafe.Sizeof(TCP6RowOwnerPID{})) } +type udpRowOwnerPIDExtractor callbackFn + // Extract will parse a row of Size() bytes pointed to by ptr func (e udpRowOwnerPIDExtractor) Extract(ptr unsafe.Pointer) { row := (*UDPRowOwnerPID)(ptr) @@ -191,6 +178,8 @@ func (udpRowOwnerPIDExtractor) Size() int { return int(unsafe.Sizeof(UDPRowOwnerPID{})) } +type udp6RowOwnerPIDExtractor callbackFn + // Extract will parse a row of Size() bytes pointed to by ptr func (e udp6RowOwnerPIDExtractor) Extract(ptr unsafe.Pointer) { row := (*UDP6RowOwnerPID)(ptr) @@ -202,21 +191,18 @@ func (udp6RowOwnerPIDExtractor) Size() int { return int(unsafe.Sizeof(UDP6RowOwnerPID{})) } -func addressIPv4(value uint32) net.IP { - address := make([]byte, 4) - machineEndiannes.PutUint32(address, value) - return net.IP(address) +func addressIPv6(s [16]byte) net.IP { + return s[:] } -func addressIPv6(s [16]byte) net.IP { - return net.IP(s[:]) +func addressIPv4(value uint32) net.IP { + return net.IP((*[4]byte)(unsafe.Pointer(&value))[:]) } -func getMachineEndiannes() binary.ByteOrder { - var buf [2]byte - *(*uint16)(unsafe.Pointer(&buf[0])) = 1 - if buf[0] == 1 { - return binary.LittleEndian - } - return binary.BigEndian +// The MIB_(TCP|UDP)_ROW_xxx structures use a 32-bit field to store ports: +// The first 16 bits contain the port in big-endian encoding +// The last 16 bits are unused. +// See links on the corresponding types in syscall_windows.go. +func uint32FieldToPort(be uint32) uint16 { + return bits.ReverseBytes16(uint16(be)) } diff --git a/packetbeat/procs/procs_windows_test.go b/packetbeat/procs/procs_windows_test.go index 4abfe359b71..4e670dcf368 100644 --- a/packetbeat/procs/procs_windows_test.go +++ b/packetbeat/procs/procs_windows_test.go @@ -21,6 +21,8 @@ package procs import ( + "bytes" + "encoding/binary" "encoding/hex" "fmt" "net" @@ -37,7 +39,7 @@ func TestParseTableRaw(t *testing.T) { pid := uint32(0xCCCCCCCC) for idx, testCase := range []struct { name string - factory extractorFactory + factory func(fn callbackFn) extractor raw string expected []portProcMapping mustErr bool @@ -110,8 +112,61 @@ func TestParseTableRaw(t *testing.T) { } } -func TestParseTableSizes(t *testing.T) { - // Make sure the structs in Golang have the expected size - assert.Equal(t, uintptr(sizeOfTCPRowOwnerPID), unsafe.Sizeof(TCPRowOwnerPID{})) - assert.Equal(t, uintptr(sizeOfTCP6RowOwnerPID), unsafe.Sizeof(TCP6RowOwnerPID{})) +func TestAddressIPv4(t *testing.T) { + // The dwLocalAddr and dwRemoteAddr members are stored as a DWORD in the same format as the in_addr structure. + // e.g. https://docs.microsoft.com/en-us/windows/win32/api/tcpmib/ns-tcpmib-mib_tcprow_owner_pid#remarks + network := binary.BigEndian + + for _, test := range []struct { + // https://docs.microsoft.com/en-us/windows/win32/api/winsock2/ns-winsock2-in_addr + a, b, c, d uint8 + }{ + {a: 1, b: 2, c: 3, d: 4}, + {a: 128, b: 64, c: 196, d: 32}, + } { + var buf bytes.Buffer + err := binary.Write(&buf, network, test) + if err != nil { + t.Errorf("failed to write %+v: %v", test, err) + continue + } + dword := *(*uint32)(unsafe.Pointer((*[4]byte)(buf.Bytes()))) + got := addressIPv4(dword) + want := net.IP{test.a, test.b, test.c, test.d} + if !got.Equal(want) { + t.Errorf("unexpected result from %+v: got:%d want:%d", test, got, want) + } + } +} + +func TestUint32FieldToPort(t *testing.T) { + // The dwLocalPort, and dwRemotePort members are in network byte order. + // e.g. https://docs.microsoft.com/en-us/windows/win32/api/tcpmib/ns-tcpmib-mib_tcprow_owner_pid#remarks + network := binary.BigEndian + + for _, test := range []struct { + port uint16 + decoy uint16 + }{ + {port: 1, decoy: 0xffff}, + {port: 2, decoy: 0xffff}, + {port: 128, decoy: 0xffff}, + {port: 256, decoy: 0xffff}, + {port: 512, decoy: 0xffff}, + {port: 512, decoy: 0xffff}, + {port: 32767, decoy: 0xffff}, + } { + var buf bytes.Buffer + err := binary.Write(&buf, network, test) + if err != nil { + t.Errorf("failed to write %+v: %v", test, err) + continue + } + dword0 := *(*uint32)(unsafe.Pointer((*[4]byte)(buf.Bytes()))) + got := uint32FieldToPort(dword0) + want := test.port + if got != want { + t.Errorf("unexpected result from %+v: got:%d want:%d", test, got, want) + } + } } diff --git a/packetbeat/procs/syscall_windows.go b/packetbeat/procs/syscall_windows.go index 5960ce32592..b7933f38dd9 100644 --- a/packetbeat/procs/syscall_windows.go +++ b/packetbeat/procs/syscall_windows.go @@ -18,21 +18,38 @@ //go:build windows // +build windows +//nolint:structcheck // Struct fields reflect Windows layout. package procs +// Add -trace to enable debug prints around syscalls. +//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go + import ( "syscall" + "unsafe" ) const ( UDP_TABLE_OWNER_PID = 1 TCP_TABLE_OWNER_PID_ALL = 5 - sizeOfDWORD = 4 - sizeOfTCPRowOwnerPID = 24 - sizeOfTCP6RowOwnerPID = 56 + sizeOfDWORD = int(unsafe.Sizeof(uint32(0))) + sizeOfTCPRowOwnerPID = int(unsafe.Sizeof(TCPRowOwnerPID{})) + sizeOfTCP6RowOwnerPID = int(unsafe.Sizeof(TCP6RowOwnerPID{})) ) +func _() { + // Make sure the structs in Go have the expected size. + + // An invalid array index indicates that the size of the Go struct does not match + // the expected size according to the Microsoft documentation. + var x [1]struct{} + _ = x[sizeOfDWORD-4] + _ = x[sizeOfTCPRowOwnerPID-24] + _ = x[sizeOfTCP6RowOwnerPID-56] +} + +// https://docs.microsoft.com/en-us/windows/win32/api/tcpmib/ns-tcpmib-mib_tcprow_owner_pid type TCPRowOwnerPID struct { state uint32 localAddr uint32 @@ -42,6 +59,7 @@ type TCPRowOwnerPID struct { owningPID uint32 } +// https://docs.microsoft.com/en-us/windows/win32/api/tcpmib/ns-tcpmib-mib_tcp6row_owner_pid type TCP6RowOwnerPID struct { localAddr [16]byte localScopeID uint32 @@ -53,12 +71,14 @@ type TCP6RowOwnerPID struct { owningPID uint32 } +// https://docs.microsoft.com/en-us/windows/win32/api/udpmib/ns-udpmib-mib_udprow_owner_pid type UDPRowOwnerPID struct { localAddr uint32 localPort uint32 owningPID uint32 } +// https://docs.microsoft.com/en-us/windows/win32/api/udpmib/ns-udpmib-mib_udp6row_owner_pid type UDP6RowOwnerPID struct { localAddr [16]byte localScopeID uint32 @@ -69,9 +89,6 @@ type UDP6RowOwnerPID struct { // GetExtendedTableFn is the prototype for GetExtendedTcpTable and GetExtendedUdpTable type GetExtendedTableFn func(pTcpTable uintptr, pdwSize *uint32, bOrder bool, ulAf uint32, tableClass uint32, reserved uint32) (code syscall.Errno, err error) -// Add -trace to enable debug prints around syscalls. -//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go - // Windows API calls //sys _GetExtendedTcpTable(pTcpTable uintptr, pdwSize *uint32, bOrder bool, ulAf uint32, tableClass uint32, reserved uint32) (code syscall.Errno, err error) = iphlpapi.GetExtendedTcpTable //sys _GetExtendedUdpTable(pTcpTable uintptr, pdwSize *uint32, bOrder bool, ulAf uint32, tableClass uint32, reserved uint32) (code syscall.Errno, err error) = iphlpapi.GetExtendedUdpTable diff --git a/packetbeat/protos/dhcpv4/dhcpv4.go b/packetbeat/protos/dhcpv4/dhcpv4.go index a719a1c8fa2..87756a2a6fe 100644 --- a/packetbeat/protos/dhcpv4/dhcpv4.go +++ b/packetbeat/protos/dhcpv4/dhcpv4.go @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. +//nolint:errcheck // All complaints are about mapstr.M puts. package dhcpv4 import ( "fmt" + "net" "strings" "github.com/insomniacslk/dhcp/dhcpv4" @@ -126,6 +128,7 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { fields["type"] = pbf.Event.Dataset fields["status"] = "OK" + mac16 := v4.ClientHwAddr() dhcpData := mapstr.M{ "op_code": strings.ToLower(v4.OpcodeToString()), "hardware_type": v4.HwTypeToString(), @@ -133,7 +136,7 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { "transaction_id": fmt.Sprintf("0x%08x", v4.TransactionID()), "seconds": v4.NumSeconds(), "flags": strings.ToLower(v4.FlagsToString()), - "client_mac": v4.ClientHwAddrToString(), + "client_mac": formatHardwareAddr(net.HardwareAddr(mac16[:v4.HwAddrLen()])), } fields["dhcpv4"] = dhcpData @@ -160,8 +163,21 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { p.log.Warnw("Failed converting DHCP options to map", "dhcpv4", v4, "error", err) } else if len(opts) > 0 { - dhcpData.Put("option", opts) + _, _ = dhcpData.Put("option", opts) } return &evt } + +// formatHardwareAddr formats hardware addresses according to the ECS spec. +func formatHardwareAddr(addr net.HardwareAddr) string { + buf := make([]byte, 0, len(addr)*3-1) + for _, b := range addr { + if len(buf) != 0 { + buf = append(buf, '-') + } + const hexDigit = "0123456789ABCDEF" + buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf]) + } + return string(buf) +} diff --git a/packetbeat/protos/dhcpv4/dhcpv4_test.go b/packetbeat/protos/dhcpv4/dhcpv4_test.go index 19614ff469b..36f686ca81d 100644 --- a/packetbeat/protos/dhcpv4/dhcpv4_test.go +++ b/packetbeat/protos/dhcpv4/dhcpv4_test.go @@ -82,7 +82,7 @@ var ( ) func TestParseDHCPRequest(t *testing.T) { - logp.TestingSetup() + _ = logp.TestingSetup() p, err := newPlugin(true, nil, procs.ProcessesWatcher{}, nil) if err != nil { t.Fatal(err) @@ -137,7 +137,7 @@ func TestParseDHCPRequest(t *testing.T) { "ip": []string{"0.0.0.0", "255.255.255.255"}, }, "dhcpv4": mapstr.M{ - "client_mac": "00:0b:82:01:fc:42", + "client_mac": "00-0B-82-01-FC-42", "flags": "unicast", "hardware_type": "Ethernet", "hops": 0, @@ -161,7 +161,8 @@ func TestParseDHCPRequest(t *testing.T) { actual := p.parseDHCPv4(pkt) if assert.NotNil(t, actual) { - publish.MarshalPacketbeatFields(actual, nil, nil) + _, err := publish.MarshalPacketbeatFields(actual, nil, nil) + assert.NoError(t, err, "marshalling packet beat fields") t.Logf("DHCP event: %+v", actual) assertEqual(t, expected, *actual) } @@ -223,7 +224,7 @@ func TestParseDHCPACK(t *testing.T) { }, "dhcpv4": mapstr.M{ "assigned_ip": "192.168.0.10", - "client_mac": "00:0b:82:01:fc:42", + "client_mac": "00-0B-82-01-FC-42", "flags": "unicast", "hardware_type": "Ethernet", "hops": 0, @@ -244,7 +245,8 @@ func TestParseDHCPACK(t *testing.T) { actual := p.parseDHCPv4(pkt) if assert.NotNil(t, actual) { - publish.MarshalPacketbeatFields(actual, nil, nil) + _, err := publish.MarshalPacketbeatFields(actual, nil, nil) + assert.NoError(t, err, "marshalling packet beat fields") t.Logf("DHCP event: %+v", actual) assertEqual(t, expected, *actual) } diff --git a/packetbeat/protos/tcp/tcp.go b/packetbeat/protos/tcp/tcp.go index 249841f991e..7665df62f3b 100644 --- a/packetbeat/protos/tcp/tcp.go +++ b/packetbeat/protos/tcp/tcp.go @@ -71,10 +71,7 @@ const ( seqGT seqCompare = 1 ) -var ( - debugf = logp.MakeDebug("tcp") - isDebug = false -) +var isDebug = false func (tcp *TCP) getID() uint32 { tcp.id++ @@ -132,7 +129,7 @@ func (stream *TCPStream) addPacket(pkt *protos.Packet, tcphdr *layers.TCP) { if mod == nil { if isDebug { protocol := conn.protocol - debugf("Ignoring protocol for which we have no module loaded: %s", + logp.Debug("tcp", "Ignoring protocol for which we have no module loaded: %s", protocol) } return @@ -172,7 +169,7 @@ func (tcp *TCP) Process(id *flows.FlowID, tcphdr *layers.TCP, pkt *protos.Packet } if isDebug { - debugf("tcp flow id: %p", id) + logp.Debug("tcp", "tcp flow id: %p", id) } if len(pkt.Payload) == 0 && !tcphdr.FIN { @@ -185,14 +182,14 @@ func (tcp *TCP) Process(id *flows.FlowID, tcphdr *layers.TCP, pkt *protos.Packet tcpSeq := tcpStartSeq + uint32(len(pkt.Payload)) lastSeq := conn.lastSeq[stream.dir] if isDebug { - debugf("pkt.start_seq=%v pkt.last_seq=%v stream.last_seq=%v (len=%d)", + logp.Debug("tcp", "pkt.start_seq=%v pkt.last_seq=%v stream.last_seq=%v (len=%d)", tcpStartSeq, tcpSeq, lastSeq, len(pkt.Payload)) } if len(pkt.Payload) > 0 && lastSeq != 0 { if tcpSeqBeforeEq(tcpSeq, lastSeq) { if isDebug { - debugf("Ignoring retransmitted segment. pkt.seq=%v len=%v stream.seq=%v", + logp.Debug("tcp", "Ignoring retransmitted segment. pkt.seq=%v len=%v stream.seq=%v", tcphdr.Seq, len(pkt.Payload), lastSeq) } return @@ -205,11 +202,11 @@ func (tcp *TCP) Process(id *flows.FlowID, tcphdr *layers.TCP, pkt *protos.Packet } gap := int(tcpStartSeq - lastSeq) - debugf("Gap in tcp stream. last_seq: %d, seq: %d, gap: %d", lastSeq, tcpStartSeq, gap) + logp.Debug("tcp", "Gap in tcp stream. last_seq: %d, seq: %d, gap: %d", lastSeq, tcpStartSeq, gap) drop := stream.gapInStream(gap) if drop { if isDebug { - debugf("Dropping connection state because of gap") + logp.Debug("tcp", "Dropping connection state because of gap") } droppedBecauseOfGaps.Add(1) @@ -224,7 +221,7 @@ func (tcp *TCP) Process(id *flows.FlowID, tcphdr *layers.TCP, pkt *protos.Packet delta := lastSeq - tcpStartSeq if isDebug { - debugf("Overlapping tcp segment. last_seq %d, seq: %d, delta: %d", + logp.Debug("tcp", "Overlapping tcp segment. last_seq %d, seq: %d, delta: %d", lastSeq, tcpStartSeq, delta) } @@ -260,7 +257,7 @@ func (tcp *TCP) getStream(pkt *protos.Packet) (stream TCPStream, created bool) { if isDebug { t := pkt.Tuple - debugf("Connection src[%s:%d] dst[%s:%d] doesn't exist, creating new", + logp.Debug("tcp", "Connection src[%s:%d] dst[%s:%d] doesn't exist, creating new", t.SrcIP.String(), t.SrcPort, t.DstIP.String(), t.DstPort) } @@ -332,7 +329,7 @@ func NewTCP(p protos.Protocols) (*TCP, error) { tcp.streams.StartJanitor(protos.DefaultTransactionExpiration) if isDebug { - debugf("tcp", "Port map: %v", portMap) + logp.Debug("tcp", "Port map: %v", portMap) } return tcp, nil diff --git a/packetbeat/sniffer/afpacket.go b/packetbeat/sniffer/afpacket.go index eac8240219c..6937b213ea8 100644 --- a/packetbeat/sniffer/afpacket.go +++ b/packetbeat/sniffer/afpacket.go @@ -23,9 +23,7 @@ import "fmt" // allocated mmap buffer is close to but smaller than target_size_mb. // The restriction is that the block_size must be divisible by both the // frame size and page size. -func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) ( - frameSize int, blockSize int, numBlocks int, err error, -) { +func afpacketComputeSize(targetSizeMb, snaplen, pageSize int) (frameSize, blockSize, numBlocks int, err error) { if snaplen < pageSize { frameSize = pageSize / (pageSize / snaplen) } else { diff --git a/packetbeat/sniffer/afpacket_linux.go b/packetbeat/sniffer/afpacket_linux.go index fc1e4dd52a6..fe77629d472 100644 --- a/packetbeat/sniffer/afpacket_linux.go +++ b/packetbeat/sniffer/afpacket_linux.go @@ -43,9 +43,7 @@ type afpacketHandle struct { device string } -func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int, - timeout time.Duration, autoPromiscMode bool) (*afpacketHandle, error, -) { +func newAfpacketHandle(device string, snaplen, block_size, num_blocks int, timeout time.Duration, autoPromiscMode bool) (*afpacketHandle, error) { var err error var promiscEnabled bool diff --git a/packetbeat/beater/worker.go b/packetbeat/sniffer/decoders.go similarity index 79% rename from packetbeat/beater/worker.go rename to packetbeat/sniffer/decoders.go index 2c7f1d7eff6..cb99ae4bfbc 100644 --- a/packetbeat/beater/worker.go +++ b/packetbeat/sniffer/decoders.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package beater +package sniffer import ( "github.com/google/gopacket/layers" @@ -29,11 +29,16 @@ import ( "github.com/elastic/beats/v7/packetbeat/protos/tcp" "github.com/elastic/beats/v7/packetbeat/protos/udp" "github.com/elastic/beats/v7/packetbeat/publish" - "github.com/elastic/beats/v7/packetbeat/sniffer" ) -func workerFactory(publisher *publish.TransactionPublisher, protocols *protos.ProtocolsStruct, watcher procs.ProcessesWatcher, flows *flows.Flows, cfg config.Config) func(dl layers.LinkType) (sniffer.Worker, error) { - return func(dl layers.LinkType) (sniffer.Worker, error) { +// Decoders functions return a Decoder able to process the provided network +// link type for use with a Sniffer. +type Decoders func(layers.LinkType) (*decoder.Decoder, error) + +// DecodersFor returns a source of Decoders using the provided configuration +// components. +func DecodersFor(publisher *publish.TransactionPublisher, protocols *protos.ProtocolsStruct, watcher procs.ProcessesWatcher, flows *flows.Flows, cfg config.Config) Decoders { + return func(dl layers.LinkType) (*decoder.Decoder, error) { var icmp4 icmp.ICMPv4Processor var icmp6 icmp.ICMPv6Processor config, err := cfg.ICMP() diff --git a/packetbeat/sniffer/file.go b/packetbeat/sniffer/file.go index 71853418fef..19667afe594 100644 --- a/packetbeat/sniffer/file.go +++ b/packetbeat/sniffer/file.go @@ -65,7 +65,7 @@ func (h *fileHandler) open() error { func (h *fileHandler) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) { data, ci, err := h.pcapHandle.ReadPacketData() if err != nil { - if err != io.EOF { + if err != io.EOF { //nolint:errorlint // io.EOF should never be wrapped. return data, ci, err } @@ -79,7 +79,7 @@ func (h *fileHandler) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) { logp.Debug("sniffer", "Reopening the file") if err = h.open(); err != nil { - return nil, ci, fmt.Errorf("Error reopening file: %s", err) + return nil, ci, fmt.Errorf("failed to reopen file: %w", err) } data, ci, err = h.pcapHandle.ReadPacketData() diff --git a/packetbeat/sniffer/sniffer.go b/packetbeat/sniffer/sniffer.go index 554e853657a..d1c4e29b018 100644 --- a/packetbeat/sniffer/sniffer.go +++ b/packetbeat/sniffer/sniffer.go @@ -42,19 +42,10 @@ type Sniffer struct { state atomic.Int32 // store snifferState - // bpf filter + // filter is the bpf filter program used by the sniffer. filter string - factory WorkerFactory -} - -// WorkerFactory constructs a new worker instance for use with a Sniffer. -type WorkerFactory func(layers.LinkType) (Worker, error) - -// Worker defines the callback interfaces a Sniffer instance will use -// to forward packets. -type Worker interface { - OnPacket(data []byte, ci *gopacket.CaptureInfo) + decoders Decoders } type snifferHandle interface { @@ -73,17 +64,12 @@ const ( // New create a new Sniffer instance. Settings are validated in a best effort // only, but no device is opened yet. Accessing and configuring the actual device // is done by the Run method. -func New( - testMode bool, - filter string, - factory WorkerFactory, - interfaces config.InterfacesConfig, -) (*Sniffer, error) { +func New(testMode bool, filter string, decoders Decoders, interfaces config.InterfacesConfig) (*Sniffer, error) { s := &Sniffer{ - filter: filter, - config: interfaces, - factory: factory, - state: atomic.MakeInt32(snifferInactive), + filter: filter, + config: interfaces, + decoders: decoders, + state: atomic.MakeInt32(snifferInactive), } logp.Debug("sniffer", "BPF filter: '%s'", filter) @@ -141,7 +127,7 @@ func New( func (s *Sniffer) Run() error { handle, err := s.open() if err != nil { - return fmt.Errorf("Error starting sniffer: %s", err) + return fmt.Errorf("failed to start sniffer: %w", err) } defer handle.Close() @@ -154,10 +140,13 @@ func (s *Sniffer) Run() error { defer f.Close() w = pcapgo.NewWriterNanos(f) - w.WriteFileHeader(65535, handle.LinkType()) + err = w.WriteFileHeader(65535, handle.LinkType()) + if err != nil { + return fmt.Errorf("failed to write dump file header to %s: %w", s.config.Dumpfile, err) + } } - worker, err := s.factory(handle.LinkType()) + decoder, err := s.decoders(handle.LinkType()) if err != nil { return err } @@ -173,28 +162,28 @@ func (s *Sniffer) Run() error { var packets int for s.state.Load() == snifferActive { if s.config.OneAtATime { - fmt.Println("Press enter to read packet") + fmt.Fprintln(os.Stdout, "Press enter to read packet") fmt.Scanln() } data, ci, err := handle.ReadPacketData() - if err == pcap.NextErrorTimeoutExpired || isAfpacketErrTimeout(err) { + if err == pcap.NextErrorTimeoutExpired || isAfpacketErrTimeout(err) { //nolint:errorlint // pcap.NextErrorTimeoutExpired is not wrapped. logp.Debug("sniffer", "timedout") continue } if err != nil { // ignore EOF, if sniffer was driven from file - if err == io.EOF && s.config.File != "" { + if err == io.EOF && s.config.File != "" { //nolint:errorlint // io.EOF should never be wrapped. return nil } s.state.Store(snifferInactive) - return fmt.Errorf("Sniffing error: %w", err) + return fmt.Errorf("sniffing error: %w", err) } if len(data) == 0 { - // Empty packet, probably timeout from afpacket + // Empty packet, probably timeout from afpacket. continue } @@ -208,7 +197,7 @@ func (s *Sniffer) Run() error { } logp.Debug("sniffer", "Packet number: %d", packets) - worker.OnPacket(data, &ci) + decoder.OnPacket(data, &ci) } return nil @@ -225,15 +214,14 @@ func (s *Sniffer) open() (snifferHandle, error) { case "af_packet": return openAFPacket(s.filter, &s.config) default: - return nil, fmt.Errorf("Unknown sniffer type: %s", s.config.Type) + return nil, fmt.Errorf("unknown sniffer type: %s", s.config.Type) } } // Stop marks a sniffer as stopped. The Run method will return once the stop // signal has been given. -func (s *Sniffer) Stop() error { +func (s *Sniffer) Stop() { s.state.Store(snifferClosing) - return nil } func validateConfig(filter string, cfg *config.InterfacesConfig) error { @@ -249,7 +237,7 @@ func validateConfig(filter string, cfg *config.InterfacesConfig) error { case "af_packet": return validateAfPacketConfig(cfg) default: - return fmt.Errorf("Unknown sniffer type: %s", cfg.Type) + return fmt.Errorf("unknown sniffer type: %s", cfg.Type) } } diff --git a/packetbeat/tests/system/test_0060_flows.py b/packetbeat/tests/system/test_0060_flows.py index 91598ffbdc0..e1d262b439e 100644 --- a/packetbeat/tests/system/test_0060_flows.py +++ b/packetbeat/tests/system/test_0060_flows.py @@ -43,8 +43,8 @@ def test_mysql_flow(self): assert len(objs) == 1 check_fields(objs[0], { 'flow.final': True, - 'source.mac': '0a:00:27:00:00:00', - 'destination.mac': '08:00:27:76:d7:41', + 'source.mac': '0A-00-27-00-00-00', + 'destination.mac': '08-00-27-76-D7-41', 'destination.ip': '192.168.33.14', 'source.ip': '192.168.33.1', 'network.transport': 'tcp', @@ -78,8 +78,8 @@ def test_memcache_udp_flow(self): assert len(objs) == 1 check_fields(objs[0], { 'flow.final': True, - 'source.mac': 'ac:bc:32:77:41:0b', - 'destination.mac': '08:00:27:dd:3b:28', + 'source.mac': 'AC-BC-32-77-41-0B', + 'destination.mac': '08-00-27-DD-3B-28', 'source.ip': '192.168.188.37', 'destination.ip': '192.168.188.38', 'network.transport': 'udp', @@ -106,8 +106,8 @@ def test_icmp4_ping(self): assert len(objs) == 1 check_fields(objs[0], { 'flow.final': True, - 'source.mac': '00:00:00:00:00:01', - 'destination.mac': '00:00:00:00:00:02', + 'source.mac': '00-00-00-00-00-01', + 'destination.mac': '00-00-00-00-00-02', 'flow.vlan': 10, 'source.ip': '10.0.0.1', 'destination.ip': '10.0.0.2', @@ -135,11 +135,11 @@ def test_icmp6_ping(self): check_fields(objs[0], { 'flow.final': True, 'flow.vlan': 10, - 'source.mac': '00:00:00:00:00:01', + 'source.mac': '00-00-00-00-00-01', 'source.ip': '::1', 'source.bytes': 70, 'source.packets': 1, - 'destination.mac': '00:00:00:00:00:02', + 'destination.mac': '00-00-00-00-00-02', 'destination.ip': '::2', 'destination.bytes': 70, 'destination.packets': 1, @@ -168,8 +168,8 @@ def test_q_in_q_flow(self): 'source.ip': '192.168.1.1', 'source.bytes': 82, 'source.packets': 1, - 'source.mac': '08:00:27:3d:25:4e', - 'destination.mac': '1c:af:f7:70:ed:7c', + 'source.mac': '08-00-27-3D-25-4E', + 'destination.mac': '1C-AF-F7-70-ED-7C', 'destination.ip': '192.168.1.2', 'network.bytes': 82, 'network.packets': 1, diff --git a/packetbeat/tests/system/test_0066_dhcp.py b/packetbeat/tests/system/test_0066_dhcp.py index 2025df30278..ec1a14b592e 100644 --- a/packetbeat/tests/system/test_0066_dhcp.py +++ b/packetbeat/tests/system/test_0066_dhcp.py @@ -18,7 +18,7 @@ def test_dhcp(self): assert objs[0]["client.ip"] == "0.0.0.0" assert objs[0]["client.port"] == 68 assert objs[0]["destination.ip"] == "255.255.255.255" - assert objs[0]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" + assert objs[0]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42" assert objs[0]["dhcpv4.flags"] == "unicast" assert objs[0]["dhcpv4.hardware_type"] == "Ethernet" assert objs[0]["dhcpv4.hops"] == 0 @@ -50,7 +50,7 @@ def test_dhcp(self): assert objs[1]["client.port"] == 68 assert objs[1]["destination.ip"] == "192.168.0.10" assert objs[1]["dhcpv4.assigned_ip"] == "192.168.0.10" - assert objs[1]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" + assert objs[1]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42" assert objs[1]["dhcpv4.flags"] == "unicast" assert objs[1]["dhcpv4.hardware_type"] == "Ethernet" assert objs[1]["dhcpv4.hops"] == 0 @@ -79,7 +79,7 @@ def test_dhcp(self): assert "event.start" in objs[2] assert objs[2]["client.ip"] == "0.0.0.0" assert objs[2]["client.port"] == 68 - assert objs[2]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" + assert objs[2]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42" assert objs[2]["dhcpv4.flags"] == "unicast" assert objs[2]["dhcpv4.hardware_type"] == "Ethernet" assert objs[2]["dhcpv4.hops"] == 0 @@ -111,7 +111,7 @@ def test_dhcp(self): assert objs[3]["client.port"] == 68 assert objs[3]["destination.ip"] == "192.168.0.10" assert objs[3]["dhcpv4.assigned_ip"] == "192.168.0.10" - assert objs[3]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" + assert objs[3]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42" assert objs[3]["dhcpv4.flags"] == "unicast" assert objs[3]["dhcpv4.hardware_type"] == "Ethernet" assert objs[3]["dhcpv4.hops"] == 0 diff --git a/testing/environments/docker/kafka/Dockerfile b/testing/environments/docker/kafka/Dockerfile index ac51d7394c4..f3fbe9f391d 100644 --- a/testing/environments/docker/kafka/Dockerfile +++ b/testing/environments/docker/kafka/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stretch +FROM debian:buster ENV KAFKA_HOME /kafka # Controls the hostname advertised within the Docker network, should generally match the container @@ -10,7 +10,7 @@ ENV KAFKA_VERSION 2.2.2 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true" ENV TERM=linux -RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat +RUN apt-get update && apt-get install -y curl openjdk-11-jre-headless netcat RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \ curl -J -L -s -f -o - https://github.com/kadwanev/retry/releases/download/1.0.1/retry-1.0.1.tar.gz | tar xfz - -C /usr/local/bin && \ diff --git a/testing/environments/docker/kafka/README.md b/testing/environments/docker/kafka/README.md index 6a7306e2423..16b6223f988 100644 --- a/testing/environments/docker/kafka/README.md +++ b/testing/environments/docker/kafka/README.md @@ -10,23 +10,23 @@ The files in the `certs` directory were generated with these commands: ```sh # create the broker's key -keytool -keystore broker.keystore.jks -storepass KafkaTest -alias broker -validity 5000 -keyalg RSA -genkey +keytool -keystore broker.keystore.jks -storepass KafkaTest -alias broker -validity 5000 -keyalg RSA -sigalg SHA256withRSA -genkey What is your first and last name? [Unknown]: kafka ... -# create a new certificate authority +# create a new certificate authority, use passphrase KafkaTest openssl req -new -x509 -keyout ca-key -out ca-cert -days 5000 # add the CA to the kafka client's trust store -keytool -keystore client.truststore.jks -storepass KafkaTest -alias CARoot -keyalg RSA -import -file ca-cert +keytool -keystore client.truststore.jks -storepass KafkaTest -alias CARoot -keyalg RSA -sigalg SHA256withRSA -import -file ca-cert # export the server certificate keytool -keystore broker.keystore.jks -storepass KafkaTest -alias broker -certreq -file broker-cert # sign it with the CA -openssl x509 -req -CA ca-cert -CAkey ca-key -in broker-cert -out broker-cert-signed -days 5000 -CAcreateserial -passin pass:KafkaTest +openssl x509 -req -CA ca-cert -CAkey ca-key -in broker-cert -out broker-cert-signed -days 5000 -CAcreateserial -passin pass:KafkaTest -sha256 # import CA and signed cert back into server keystore keytool -keystore broker.keystore.jks -storepass KafkaTest -alias CARoot -import -file ca-cert diff --git a/testing/environments/docker/kafka/certs/broker-cert b/testing/environments/docker/kafka/certs/broker-cert index 3a7d9e2498a..d40676da0a6 100644 --- a/testing/environments/docker/kafka/certs/broker-cert +++ b/testing/environments/docker/kafka/certs/broker-cert @@ -1,18 +1,18 @@ -----BEGIN NEW CERTIFICATE REQUEST----- -MIIC3zCCAccCAQAwajEQMA4GA1UEBhMHVW5rbm93bjEQMA4GA1UECBMHVW5rbm93 -bjEQMA4GA1UEBxMHVW5rbm93bjEQMA4GA1UEChMHVW5rbm93bjEQMA4GA1UECxMH -VW5rbm93bjEOMAwGA1UEAxMFa2Fma2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQCH8VYN9FMHXjnLUwT0AJDKM0u/jXE0ng1UfWPVQaVI+Eny+vmf1zDm -d/AoqXaYKzVNvyRXCy1BZGaLVA3go1U7+tVjtniuLTmveE07PuX4w9/ukZPKlUxf -KCjYCmh38BeYiJA2inaxScDO2hxHfB2pulsM+l9+q0NMXFe6RSUAKS0pAeY8KLz9 -yWg9hfq6JPuPT14HZmyxLn+1SwRbZZ+TQjlAHfZFpu/igg6cif/ez30z5Gqci+2i -VPlwl9peEsaXn5wbuP6J2Uo6dMoGiFyxFdGCWVWP9WDncvfYKJwQs09QdbFLxAst -BYSmOTszUP+h0SohaxpdC4AOcJxs+MwhAgMBAAGgMDAuBgkqhkiG9w0BCQ4xITAf -MB0GA1UdDgQWBBRFzbnwQXp+h4xE233eH3D+KfozxTANBgkqhkiG9w0BAQsFAAOC -AQEAQti4SPU8KfSoeLbLUic7UciVmwO0TZtiG+Y6fCTdRm7SYovg2zXH576ERClf -JQCzUuMH1Fi6k5adhMUxopJrVirZWOANoffe3yY/PUuFPMv5rvjmG7JqRNloNFYC -4Jah/XeITkw3BcwYxvY3lOZeXgBoRI+PwaD4JNHYf9ruc8cxY59lbWGCQOdbWYuk -ex/Y/rdmiv1cZpVAYY3VkdUnISXf4eePz4+hUdyuNGYt8Rh/dCj0D/1Xdo9jguUw -IWihuXNfH5hBzBp2hX49tCa7j8stOQW6+AS+ysUBRseFNnsu9j95PD+ue9GU5ZLR -mQzlkeZcfimH796e6XF81oCDkA== +MIIC0jCCAboCAQAwXTELMAkGA1UEBhMCVVMxDjAMBgNVBAgTBWthZmthMQ4wDAYD +VQQHEwVrYWZrYTEOMAwGA1UEChMFa2Fma2ExDjAMBgNVBAsTBWthZmthMQ4wDAYD +VQQDEwVrYWZrYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL8W8cV8 +IPoqY7kyNdGeUCSS6DXy/UoNcTVaVcxHytDDske6QP5VwVQmhJDxcvpCVIr5me2o +UqSHlk60ikCkVbgspDLPg8scchXY8jxN4uVzcAIQqKOtJ02unLaSirO9uLuxq3mw +Tg/TLZ8Ny9ytPOoE0feZKOL8kE5B3ar+IzJWiVxvTpdbM1FxMd0JIE1AGIGrwQPv +OLv3mhYX/SsnbCty5PESp1dpfrvtBDDYv2AWkfJQuSlzspRHeXbRdAmkhxa1+RF0 +qovgfNXzF9LkMySm9YGSqD8WHtw7hl0PwG0vteJLH4dR3pccYQWoRiLYmPlCeg/t +ZLvZxWkiIIJjuXECAwEAAaAwMC4GCSqGSIb3DQEJDjEhMB8wHQYDVR0OBBYEFDny +LLWik9HtleB+eQgUuFRdP4J5MA0GCSqGSIb3DQEBCwUAA4IBAQBtG9QxU1i72aMh +S+dBbWiSI1AsBiiCzisV9J5Yj0BdMbzLtUG0rAT4knC0jiGEykU+1SV20M7cG22v +TRbXjvk9thVjuKlCMeeVYTmGACSuMFNhP1yje9bf8ohlP22WCfhAUqO4uCdQj8yT +QvZeO7PrdJxxSIG8GDgSFf/vdPoBzI1LUYqGD62JSyGVr+iMt0L1O/yHzYJCl/ho +ItN5xRQLoZITlrUTSzkPacU8fR2vBjv7h6/pTzlzJ1fbHK3yS34HojMdc7v+Q0Qb +yaOomDnX++/W/2vS1LIocK0M2/qX9Nt6eyIVe1o+dsZYirXCJRcWZ4U0L71qIvMf +VqNymL1T -----END NEW CERTIFICATE REQUEST----- diff --git a/testing/environments/docker/kafka/certs/broker-cert-signed b/testing/environments/docker/kafka/certs/broker-cert-signed index b023f3c146d..b52a642064e 100644 --- a/testing/environments/docker/kafka/certs/broker-cert-signed +++ b/testing/environments/docker/kafka/certs/broker-cert-signed @@ -1,18 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIC8zCCAdsCCQC1GCJdAf28SzANBgkqhkiG9w0BAQUFADANMQswCQYDVQQGEwJV -UzAeFw0yMTEwMjEyMDM0MTBaFw0zNTA2MzAyMDM0MTBaMGoxEDAOBgNVBAYTB1Vu -a25vd24xEDAOBgNVBAgTB1Vua25vd24xEDAOBgNVBAcTB1Vua25vd24xEDAOBgNV -BAoTB1Vua25vd24xEDAOBgNVBAsTB1Vua25vd24xDjAMBgNVBAMTBWthZmthMIIB -IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAh/FWDfRTB145y1ME9ACQyjNL -v41xNJ4NVH1j1UGlSPhJ8vr5n9cw5nfwKKl2mCs1Tb8kVwstQWRmi1QN4KNVO/rV -Y7Z4ri05r3hNOz7l+MPf7pGTypVMXygo2Apod/AXmIiQNop2sUnAztocR3wdqbpb -DPpffqtDTFxXukUlACktKQHmPCi8/cloPYX6uiT7j09eB2ZssS5/tUsEW2Wfk0I5 -QB32Rabv4oIOnIn/3s99M+RqnIvtolT5cJfaXhLGl5+cG7j+idlKOnTKBohcsRXR -gllVj/Vg53L32CicELNPUHWxS8QLLQWEpjk7M1D/odEqIWsaXQuADnCcbPjMIQID -AQABMA0GCSqGSIb3DQEBBQUAA4IBAQCMGbXC2YdC9+jJjUvuEJIQGwpapJ5Dejng -cnvE//+x8A4W9vC7OJUHcML2GGQIrgvYWlmsCEWX1lJtcVIbqkTqq9Sq99htdMfM -ay4fJB/ey005bhcbEP+19342HkmoOUkEg7qGWZhhL05y0m1vxKvKSUX3p+4TyW1Y -AheRbb9j41Ld3E8+COGwqIWpMNfsGjLqWjUIajemFH91Eo2FFvshM/5ly12GZEil -ivmUqSzV7o6ri0V7DZ5NPOSXEbiMQj5FfmImqXbo7JtBqM/H9S2yAPXZBfAloVNv -XvjG0dY8cnYwGL5MSRiZEuJdimptWnMzFXbD8zyRxSIUMpbDcHNf +MIIDTDCCAjQCCQD48GlXfeHoqzANBgkqhkiG9w0BAQsFADBzMQswCQYDVQQGEwJV +UzEOMAwGA1UECAwFa2Fma2ExDjAMBgNVBAcMBWthZmthMQ4wDAYDVQQKDAVrYWZr +YTEOMAwGA1UECwwFa2Fma2ExDjAMBgNVBAMMBWthZmthMRQwEgYJKoZIhvcNAQkB +FgVrYWZrYTAeFw0yMjA4MDQxOTExMDZaFw0zNjA0MTIxOTExMDZaMF0xCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIEwVrYWZrYTEOMAwGA1UEBxMFa2Fma2ExDjAMBgNVBAoT +BWthZmthMQ4wDAYDVQQLEwVrYWZrYTEOMAwGA1UEAxMFa2Fma2EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC/FvHFfCD6KmO5MjXRnlAkkug18v1KDXE1 +WlXMR8rQw7JHukD+VcFUJoSQ8XL6QlSK+ZntqFKkh5ZOtIpApFW4LKQyz4PLHHIV +2PI8TeLlc3ACEKijrSdNrpy2koqzvbi7sat5sE4P0y2fDcvcrTzqBNH3mSji/JBO +Qd2q/iMyVolcb06XWzNRcTHdCSBNQBiBq8ED7zi795oWF/0rJ2wrcuTxEqdXaX67 +7QQw2L9gFpHyULkpc7KUR3l20XQJpIcWtfkRdKqL4HzV8xfS5DMkpvWBkqg/Fh7c +O4ZdD8BtL7XiSx+HUd6XHGEFqEYi2Jj5QnoP7WS72cVpIiCCY7lxAgMBAAEwDQYJ +KoZIhvcNAQELBQADggEBAACMzkWO0HjgnMUCuCJwNbG9/ZBA3gHeV5erBspYF/9Z +bPVvRzCAvi5VgGRefosk+Q2dT4v/BIpOvIdmHQu4IUwulDz6ICBDaAlttKBEKWwU +nKfmvRqxfphnMx2QoX+ZsInStCj7ERnYLCrOHGJrDOuJ3EfubDOqOnotkDXjSxkc +cAk2Bt5UshDFerCaRZ8kTSB1U5JMWVnAUwyDbyN43iu2EL0hDc5klvjcaIXsodug +d22GAhwnFipE+UB9sztwS3JXXAgX4r7BCJeenUAKr2bZQLL7yNYw1TDll01I/z9m ++SHnz0p+fHJva4352Spv1HJXSaForJ5SKSr85UqGYk0= -----END CERTIFICATE----- diff --git a/testing/environments/docker/kafka/certs/broker.keystore.jks b/testing/environments/docker/kafka/certs/broker.keystore.jks index aa03364e3ac..b243d544e07 100644 Binary files a/testing/environments/docker/kafka/certs/broker.keystore.jks and b/testing/environments/docker/kafka/certs/broker.keystore.jks differ diff --git a/testing/environments/docker/kafka/certs/ca-cert b/testing/environments/docker/kafka/certs/ca-cert index 725647f9d8b..f0bb7589be5 100644 --- a/testing/environments/docker/kafka/certs/ca-cert +++ b/testing/environments/docker/kafka/certs/ca-cert @@ -1,16 +1,21 @@ -----BEGIN CERTIFICATE----- -MIICljCCAX4CCQD+dvzut8IfyTANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJV -UzAeFw0yMTEwMjEyMDMyMDJaFw0zNTA2MzAyMDMyMDJaMA0xCzAJBgNVBAYTAlVT -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlR6sKchCTM0qdrjAdWqR -BmSLfHHe+LB43B0T/+3Y1fporzg2eZC1dPCf7TXLzL92NOlJ+JQCsfb160gKTGfb -7+z2jm+vumbYlKVffsD7MjNdW8SDu9hfMa2DyTY742n3R/X8pc4VK0fdlTQx22Zp -aIA+XwD6hHxZQS9PHVNwTFUoPkP4jevcFANwjLUBgy3dPK0iWdVILnaAwEBg82z3 -zWRJ7I4Eg6KS+GtwZPovhiHqcJpz7QPrmggCglL8q0YZQrVrYNucRV1sjPAhEfTA -Sh7Z0UVYdx5+jJq7MyslBqzEM0OrmKrldrTHOAo9+cTc1GiKGRBhVei2R2fP2XAC -HQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA37KiWMR6SZRmlLKV7hP9/9H1cL1FJ -OPa5MKcwh8Q38IRALCF5SlxxOByP8O01ZInkWjR3jJbMc/k4RwxQXfzYDvB4jleU -MyX63qekIsxFdUn+fzt+wA0xb7tOPGVUbM6QI++YH28p8yzSdY/bXrjRweQuVRC0 -B+0zMijI1uU6GRME9+e1OLsN5rDzCFEJUra/+UDc23BTOjC6Az00UKpOGv6oAqg8 -iuCOeVCRVPtd7mGJK1dGW3WXV3pbsu4EvfXve9qFFV/7d811JNBjnhF5lFN2JGVs -Ka9JebJ8EKWff6Ns14FJ2cOG3tx7KuWcnfTdma/mH4PeGoU1Og5Ln/ea +MIIDYjCCAkoCCQDUy30S9G2MXjANBgkqhkiG9w0BAQsFADBzMQswCQYDVQQGEwJV +UzEOMAwGA1UECAwFa2Fma2ExDjAMBgNVBAcMBWthZmthMQ4wDAYDVQQKDAVrYWZr +YTEOMAwGA1UECwwFa2Fma2ExDjAMBgNVBAMMBWthZmthMRQwEgYJKoZIhvcNAQkB +FgVrYWZrYTAeFw0yMjA4MDQxOTEwNTNaFw0zNjA0MTIxOTEwNTNaMHMxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVrYWZrYTEOMAwGA1UEBwwFa2Fma2ExDjAMBgNVBAoM +BWthZmthMQ4wDAYDVQQLDAVrYWZrYTEOMAwGA1UEAwwFa2Fma2ExFDASBgkqhkiG +9w0BCQEWBWthZmthMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAugme +4arqId/pSW3GjWIUL/JCf6kfbDj1TCQUZL/uvLMtPiEG2CbCAQSr4HHtWT/3B+Xr +QKOjSIaxEegE4T9nIB0gNE2h2pVQWfOb3QcAWuNbZnS7oo46y+QKIOgcWFimUkHh +KxqKDBYWA1ttBJTMDv6IXP4F3+/r25/A1MoWxLFxStoGAUo8rjYRdg62lhjOeTMA +3+gTbkXfWDZDFcdRa+Vy6i6CFFeYduX83JsDsvyvX5udOJ0iw2oSdF+LEbo+w+GB +i1+AEPIHs+qybkWp7a1SCjyO6pBTsslQ/di61F4K/N4GYozl/aM+jpESleqJ+Jjv +9lUONzyOeX5pD5M2RwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAv+ziF9/MFwfZG +J/CDVIQptfjNRhsC/MRHf/8QhT9CqUgIMtSBAVIEiJOJ6pYx8AHnHFUC7FgU9as8 +CGSnWeBbzkQpKoOceG//a3XUPXFgsS7as96vGsm1oCvxIQlvWlXk8VNm5h4em6rh +kJD+SzmJ1ROOwM9LQTG669eKpPR65XPgeo85I+SfuwaNWELR4Xep3rHIOGYVRZhl +Z1ctPAqFIKGYdtFw7fJZL0HnP+peNmc1WCC/WRpFtnUV1GKPkPuy9jzJ6haIdlX4 +1c5KEOyLEjJvR5wdSYP/fivIga8sgGedlU2YLZSuqNoZBNutpwgo3ntdXLRZ39VV +p8T3AkOI -----END CERTIFICATE----- diff --git a/testing/environments/docker/kafka/certs/ca-cert.srl b/testing/environments/docker/kafka/certs/ca-cert.srl index 3336d99a9a3..75591a0f2db 100644 --- a/testing/environments/docker/kafka/certs/ca-cert.srl +++ b/testing/environments/docker/kafka/certs/ca-cert.srl @@ -1 +1 @@ -B518225D01FDBC4B +F8F069577DE1E8AB diff --git a/testing/environments/docker/kafka/certs/ca-key b/testing/environments/docker/kafka/certs/ca-key index 63ab92b23aa..bb8d0e55818 100644 --- a/testing/environments/docker/kafka/certs/ca-key +++ b/testing/environments/docker/kafka/certs/ca-key @@ -1,30 +1,30 @@ -----BEGIN ENCRYPTED PRIVATE KEY----- -MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIPiUp46K/yl8CAggA -MB0GCWCGSAFlAwQBKgQQ3vI4jI41do9rQtAyc+JwfwSCBNDDDv3uSHZ/12ACMLyu -vhrz3en62CT4ooeZKjpep4H/s+2pfqfg5bDeQUguNUo2zsy0EKeK2rIBVUG6KWIa -DWvnl0JaM/TrtGg1pMZMIFPowaf1mNcVcvN5IWEgLgoT+SY8lLtmVqIdU1d2F6nV -t+7JEpktXA5ThS0FbBpW0XI+kG9W7Ln9YPjGLbjcPayJQX1yzWJYEUfmm8lJl7cT -h5V94sB2KP9pwAP/SqElt2QK1BhStUVv3ezp6TT82PETi3No0Uh+oWaxltjPCr4z -5MN/tTDJQGc3llDIrZT+umZcgB6DBsc+nXjrlAHWPeuhIcjNJGs3V0xazQs60M6n -ldhcJH453Muwtp40VDkT21plVPwUrwQX/6gIWIHnyvK44sRG4NvmG+4NBA8V4TdK -AKhkhYTYS+sAUDsAFo408OXvpdGy7G2/cZn+r2frLHLxUU2peqRFP7YqLVs2sdez -sFyt6ZMSAh8UZDYK9kpyQMoeYj7Az14kMKIlE0JADsd3Mn8S/QJrrKWQzQhQVz6O -0rpaGnIM3cICgTK7gTlK+lDIbqAmCYnFLQsU9rHIpzVMkx2iYEId+YNbxodHpFPa -MCz6HU8qI9Tv9JIOfJKdE7tvlSnR89usOU/z+NSGqKm1dhYjG1BNI7wk8/mgMxOg -9BAujodmGvFpMPba84+QT/AtTy9YMMi4Z0H7BKHGD7HwSOTx7kP9hMz/sVVnKxfO -8C9gE91D4enrpQXu7J5JU07LCWSNLiZEegbdKvjBz5Cvfj5LPhazTLYuuU0KNIP9 -MjrgodrSp1LgESAA7z8qKUyhX2Z6uO0q1Q5OUFgGNEWXSYLplhWrvftPqdV0YAFI -4y794sojVBBnHYo+Lm5gugm4cg6bLk/YY3ScQqPYEUwO1LZSMUoB9ixLHUYY48ND -xbevM9V8vLgb6Q46zTCYPxwYfxNlcWxeQjwbVEaha5n2Sgu0dmrG/+LjrEwYtHY7 -zPdTbl28OyvXDqvilXcDQS8ZQBwqkZ00pg9fokElztgVIMp4cbtBTCiqipfNBGJg -ALEu/lFNlGjvv4iwOdx/yhVjFt2Ri3ViTEoTJ3wAh3o4wh/o8wluNb3bMgfKzw61 -/WptUvLnqKIGQ0xZtunxG9WHIpc8oTRZMMUgLnoVzJvdU9cONT5GER9WuQbwXmEE -ytIx9tVq4cb3CoJhynrL9cjGp716nBkx534gyu5N21elb8npk1XAHd6AHUViun0J -TnVHPwSSLN7naaiMKS+8KaknAdjvKCIUytLSRpRb4rkoqD/7MlYlMTAPF5IX6/Xj -fVfR8HKWtkvqhAM4lQ57zwGlpXifGM5Vi/Dq8JYcTOIHIzggbhfi+WVwWJ+SJVp3 -FQBSvyJ0XFV8piuP6J1PB6zXLioRiUMDbrl0Hmwo4spLswRsZ6D/6QuNUeNN2Lh1 -ZqtkAHWnIll1nviSEWPxiu0lA9ZwfPP1t+H0UkVi8JBUCrTh0gyr2e/CGZAd1GoP -/LnvaRntmqytavI65NlPPlvF9S7enjeEkxvtqhAIuU9nTMORnmpXX+xStfm/AtQp -2UNklwWW6bwPhMF9w+FnuJoK7mrQ5DphsZNcTly1RQ0uQkT6yrzWK5MNmLRiNOez -OmM968GQKexUL9r0BmFi7T00rQ== +MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIJtcnwRIUCboCAggA +MB0GCWCGSAFlAwQBKgQQK9FMWVV6SWkfBMh1PRRkuQSCBNBe9AXT361DtUIHtvu4 +cD59bu/T/Jjt05nVr3h+bIMvkUUFF0+EVGEFq05c5eSoOcNFEjrToabv+HjqmhbQ +xgArBcKlmVKktApgcHZ6yoTnrunJiJpAj1z2fIE+nyn7E2yNp6CmCTNIZwLD2diC +Wi/hJ5xES11n90J0/+LqxxnwRpckZ/7IHo4sAAFVBn0vt9ZjLDVmYir7tbSdwfT4 +zFdVek32eVHAosZKXOQOP4LM7BGc2smfYHWhkOQ45Tl/xUKk9KdLIFOJFmxYM16B +1FMT/1WklMbMqI5ZIqjVFH/K7D98zvyWEIBGQTYUE9yx7oqNWgr6gU3V8HHERUam +UsSXEf/0YmoWgy1FrG1H7IdmQa1bayayhu9oFXAgpEP3H1Y40/12TwEiX13dD6ze +u0DTPpZZ9b1aU8SVcYl4iCBLqLlUiHB8/TsAS2Zfdu/WXzlqyBhndh2QJMoOcisX +0fN+nE3qRGpVjXu5GTK8qByjw6G8c/JAB6+/jIZBq/7Qq0GLA8+MHwkWU8Abeycf +acu1ElJwWlzJEb6tKC6Riydvucb2t6qM+xrAJQDB0A9y6vGY9VeadiMpZZrxPC8u +g8RDCmbIuQwfoxbWy5SjrE9vhp77SzBncXexoKAjFQNzZHk16M97iPpuEwvVwLvC +PHCcHFszl/TDJP3GEY13jAh9IfjqoUn8/6qEZjaNO0qcsl1UfcfAoqLCC7aOwkzg +4a+2QQEE0j/AWWqZwtYCznaqGnnLUaL4Q6Kcyd3VxleP0XzC7cEcbj8XuPcG7mVP +ciV9bv9TnCkaf0cjNjf6muj59BwbkjUHllcuEyXKl++2DfPjlFbya8W1xk65bOjW +pgHNawuL6eEyKbfXqQZG02hfG3IL/qJaW7gYWXXX1V0fBbgDboxYilQ3HhbcprPs +9hlHGh5vZqv2TzPxcqyeAzSLAJnziDx+1SCrg2VsTHZEv0xGfOUTtReouygayY1o +FyRs0N4SG7dsqhAK9zjkbUwkWsAwshgePWkW4F0v+fD7vdKf1Rkp6k5YiW8x5f5i +N2idRxECOP4xjMZDx1Np1kr1KE2qFZkb8TINHnVtb0OCxl0kZRTvbDGCITpZOQPl +t3iU+u3YLFeLRhdwwGCJbdccLY/1O0fLNTKRqgugoHkI7dC+d7DR4ObIu67eRPN/ ++HrOrGt7kvcmMIwyiCVP24xZXz0Jz7jLEhXy4E0TxYwOrd1PFIqRXkRLsr6WsDmN +FtumzvMDJ3JyOloWlyHY7p8n+6ywaXHFjtirmQzZ4NxXMsACi0oJrBj4Eaa8O9J5 +vulDUjiqkX//FAdlaQCQUIdxY/e70BdGuX5j18CrzqdqXxBMeMRxjeWTp4nb+bjv +JJiXLx0VfKZCP7usmc61GukpV30pHsbi0EOX8fW35yx6sqWgoIHBCCet0V0Nml1j +dQk1ZZfBO2t6whH82Cz726F2tR68uIIWGEZOCz3hRnXdUKDIarVZdst3N3A+HbIl +ibrD6RF+UHf0y0iG6Oadin2V8lLu/Enq9zO3XIw4Xby6MicVsADj3+AMDRxj28VC +tPpGw/ds4x4fSRStSLwd6IRk2oIsYN29xsvkd2EfEMYZPLY2kNSuMusFQTdvFkD9 +CE6tLMjpd3Htj49XZ3F+nFlxjw== -----END ENCRYPTED PRIVATE KEY----- diff --git a/testing/environments/docker/kafka/certs/client.truststore.jks b/testing/environments/docker/kafka/certs/client.truststore.jks index 7b18b3f645e..1fc6cb1373c 100644 Binary files a/testing/environments/docker/kafka/certs/client.truststore.jks and b/testing/environments/docker/kafka/certs/client.truststore.jks differ diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index c3e30de80dc..31505d7a57f 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-31269fd2-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.5.0-fedc3e60-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.4.0-31269fd2-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.5.0-fedc3e60-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-31269fd2-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.5.0-fedc3e60-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index 17ac1e46230..339fdbc1a2c 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C winlogbeat check; - make -C winlogbeat update; - make -C x-pack/winlogbeat check; - make -C x-pack/winlogbeat update; - make check-no-changes; - stage: checks crosscompile: make: "make -C winlogbeat crosscompile" stage: mandatory @@ -44,12 +36,11 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2012-r2" stage: extended_win - # See https://github.com/elastic/beats/issues/32468 - # windows-11: - # mage: "mage build unitTest" - # platforms: ## override default labels in this specific stage. - # - "windows-11" - # stage: extended_win + windows-11: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-11" + stage: extended_win windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index 585bb23c4f1..ad98229b253 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -332,7 +332,8 @@ func (l *winEventLog) Read() ([]Record, error) { if r.Message == "" { r.Message, err = l.message(h) if err != nil { - logp.Err("%s error salvaging message: %v", l.logPrefix, err) + logp.Warn("%s error salvaging message (event id=%d qualifier=%d provider=%q created at %s will be included without a message): %v", + l.logPrefix, r.EventIdentifier.ID, r.EventIdentifier.Qualifiers, r.Provider.Name, r.TimeCreated.SystemTime, err) } } records = append(records, r) diff --git a/winlogbeat/sys/wineventlog/bookmark.go b/winlogbeat/sys/wineventlog/bookmark.go index 85834e0fada..af6b0d1deff 100644 --- a/winlogbeat/sys/wineventlog/bookmark.go +++ b/winlogbeat/sys/wineventlog/bookmark.go @@ -21,9 +21,9 @@ package wineventlog import ( + "fmt" "syscall" - "github.com/pkg/errors" "golang.org/x/sys/windows" "github.com/elastic/beats/v7/winlogbeat/sys" @@ -42,8 +42,8 @@ func (b Bookmark) XML() (string, error) { var bufferUsed uint32 err := _EvtRender(NilHandle, EvtHandle(b), EvtRenderBookmark, 0, nil, &bufferUsed, nil) - if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { - return "", errors.Wrap(err, "failed to determine necessary buffer size for EvtRender") + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // Bad linter! This is always errno or nil. + return "", fmt.Errorf("failed to determine necessary buffer size for EvtRender: %w", err) } bb := sys.NewPooledByteBuffer() @@ -52,7 +52,7 @@ func (b Bookmark) XML() (string, error) { err = _EvtRender(NilHandle, EvtHandle(b), EvtRenderBookmark, uint32(bb.Len()), bb.PtrAt(0), &bufferUsed, nil) if err != nil { - return "", errors.Wrap(err, "failed to render bookmark XML") + return "", fmt.Errorf("failed to render bookmark XML: %w", err) } return sys.UTF16BytesToString(bb.Bytes()) diff --git a/winlogbeat/sys/wineventlog/format_message.go b/winlogbeat/sys/wineventlog/format_message.go index 642eaa69965..f97024b663c 100644 --- a/winlogbeat/sys/wineventlog/format_message.go +++ b/winlogbeat/sys/wineventlog/format_message.go @@ -87,9 +87,13 @@ func evtFormatMessage(metadataHandle EvtHandle, eventHandle EvtHandle, messageID // Get a buffer from the pool and adjust its length. bb := sys.NewPooledByteBuffer() defer bb.Free() + // The documentation for EventFormatMessage specifies that the buffer is + // requested "in characters", and the buffer itself is LPWSTR, meaning the + // characters are WCHAR so double the value. + // https://docs.microsoft.com/en-us/windows/win32/api/winevt/nf-winevt-evtformatmessage bb.Reserve(int(bufferUsed * 2)) - err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, uint32(bb.Len()), bb.PtrAt(0), &bufferUsed) + err = _EvtFormatMessage(metadataHandle, eventHandle, messageID, valuesCount, valuesPtr, messageFlag, bufferUsed, bb.PtrAt(0), &bufferUsed) switch err { //nolint:errorlint // This is an errno or nil. case nil: // OK diff --git a/winlogbeat/sys/wineventlog/iterator.go b/winlogbeat/sys/wineventlog/iterator.go index 7cfb2b7cc38..bd547de183e 100644 --- a/winlogbeat/sys/wineventlog/iterator.go +++ b/winlogbeat/sys/wineventlog/iterator.go @@ -21,9 +21,10 @@ package wineventlog import ( + "errors" + "fmt" "sync" - "github.com/pkg/errors" "golang.org/x/sys/windows" ) @@ -148,7 +149,7 @@ func (itr *EventIterator) moreHandles() bool { var numReturned uint32 err := itr.evtNext(itr.subscription, batchSize, &itr.handles[0], 0, 0, &numReturned) - switch err { + switch err { //nolint:errorlint // Bad linter! This is always errno or nil. case nil: itr.lastErr = nil itr.active = itr.handles[:numReturned] @@ -159,8 +160,7 @@ func (itr *EventIterator) moreHandles() bool { itr.subscription.Close() itr.subscription, err = itr.subscriptionFactory() if err != nil { - itr.lastErr = errors.Wrap(err, "failed in EvtNext while trying to "+ - "recover from RPC_S_INVALID_BOUND error") + itr.lastErr = fmt.Errorf("failed in EvtNext while trying to recover from RPC_S_INVALID_BOUND error: %w", err) return false } @@ -168,9 +168,7 @@ func (itr *EventIterator) moreHandles() bool { batchSize = batchSize / 2 continue } else { - itr.lastErr = errors.Wrap(err, "failed in EvtNext (try "+ - "reducing the batch size or providing a subscription "+ - "factory for automatic recovery)") + itr.lastErr = fmt.Errorf("failed in EvtNext (try reducing the batch size or providing a subscription factory for automatic recovery): %w", err) } default: itr.lastErr = err diff --git a/winlogbeat/sys/wineventlog/iterator_test.go b/winlogbeat/sys/wineventlog/iterator_test.go index 03ded9b3f40..1e7e0498e14 100644 --- a/winlogbeat/sys/wineventlog/iterator_test.go +++ b/winlogbeat/sys/wineventlog/iterator_test.go @@ -32,7 +32,7 @@ import ( ) func TestEventIterator(t *testing.T) { - logp.TestingSetup() + logp.TestingSetup() //nolint:errcheck // Not needed. writer, tearDown := createLog(t) defer tearDown() diff --git a/winlogbeat/sys/wineventlog/metadata_store.go b/winlogbeat/sys/wineventlog/metadata_store.go index 3310ea353c6..c6b5fefd1fe 100644 --- a/winlogbeat/sys/wineventlog/metadata_store.go +++ b/winlogbeat/sys/wineventlog/metadata_store.go @@ -21,12 +21,12 @@ package wineventlog import ( + "fmt" "strconv" "strings" "sync" "text/template" - "github.com/pkg/errors" "go.uber.org/multierr" "github.com/elastic/beats/v7/winlogbeat/sys" @@ -299,7 +299,7 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E // publisher metadata is unavailable or is out of sync with the events. event, err := winevent.UnmarshalXML([]byte(xml)) if err != nil { - return nil, errors.Wrap(err, "failed to unmarshal XML") + return nil, fmt.Errorf("failed to unmarshal XML: %w", err) } em := &EventMetadata{ @@ -404,8 +404,7 @@ func (em *EventMetadata) initEventMessage(itr *EventMetadataIterator, publisher msg, err := getMessageString(publisher, NilHandle, messageID, templateInserts.Slice()) if err != nil { - return errors.Wrapf(err, "failed to get message string using message "+ - "ID %v for for event ID %v", messageID, em.EventID) + return fmt.Errorf("failed to get message string using message ID %v for for event ID %v: %w", messageID, em.EventID, err) } return em.setMessage(msg) @@ -419,8 +418,7 @@ func (em *EventMetadata) setMessage(msg string) error { Delims(leftTemplateDelim, rightTemplateDelim). Funcs(eventMessageTemplateFuncs).Parse(msg) if err != nil { - return errors.Wrapf(err, "failed to parse message template for "+ - "event ID %v (template='%v')", em.EventID, msg) + return fmt.Errorf("failed to parse message template for event ID %v (template='%v'): %w", em.EventID, msg, err) } // One node means there were no parameters so this will optimize that case diff --git a/winlogbeat/sys/wineventlog/metadata_store_test.go b/winlogbeat/sys/wineventlog/metadata_store_test.go index caa278cb987..8eb2b82d203 100644 --- a/winlogbeat/sys/wineventlog/metadata_store_test.go +++ b/winlogbeat/sys/wineventlog/metadata_store_test.go @@ -29,7 +29,7 @@ import ( ) func TestPublisherMetadataStore(t *testing.T) { - logp.TestingSetup() + logp.TestingSetup() //nolint:errcheck // Not needed. s, err := NewPublisherMetadataStore( NilHandle, diff --git a/winlogbeat/sys/wineventlog/publisher_metadata.go b/winlogbeat/sys/wineventlog/publisher_metadata.go index ecbbf8f7b65..559c547a8c9 100644 --- a/winlogbeat/sys/wineventlog/publisher_metadata.go +++ b/winlogbeat/sys/wineventlog/publisher_metadata.go @@ -21,10 +21,10 @@ package wineventlog import ( + "fmt" "os" "syscall" - "github.com/pkg/errors" "go.uber.org/multierr" "golang.org/x/sys/windows" ) @@ -59,7 +59,7 @@ func NewPublisherMetadata(session EvtHandle, name string) (*PublisherMetadata, e handle, err := _EvtOpenPublisherMetadata(session, publisherName, logFile, 0, 0) if err != nil { - return nil, errors.Wrap(err, "failed in EvtOpenPublisherMetadata") + return nil, fmt.Errorf("failed in EvtOpenPublisherMetadata: %w", err) } return &PublisherMetadata{ @@ -79,7 +79,7 @@ func (m *PublisherMetadata) stringProperty(propertyID EvtPublisherMetadataProper case nil: return "", nil default: - return "", errors.Errorf("unexpected data type: %T", v) + return "", fmt.Errorf("unexpected data type: %T", v) } } @@ -94,7 +94,7 @@ func (m *PublisherMetadata) PublisherGUID() (windows.GUID, error) { case nil: return windows.GUID{}, nil default: - return windows.GUID{}, errors.Errorf("unexpected data type: %T", v) + return windows.GUID{}, fmt.Errorf("unexpected data type: %T", v) } } @@ -172,20 +172,20 @@ func NewMetadataKeywords(publisherMetadataHandle EvtHandle) ([]MetadataKeyword, arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) if !ok { - return nil, errors.Errorf("unexpected handle type: %T", v) + return nil, fmt.Errorf("unexpected handle type: %T", v) } defer arrayHandle.Close() arrayLen, err := EvtGetObjectArraySize(arrayHandle) if err != nil { - return nil, errors.Wrap(err, "failed to get keyword array length") + return nil, fmt.Errorf("failed to get keyword array length: %w", err) } var values []MetadataKeyword for i := uint32(0); i < arrayLen; i++ { md, err := NewMetadataKeyword(publisherMetadataHandle, arrayHandle, i) if err != nil { - return nil, errors.Wrapf(err, "failed to get keyword at array index %v", i) + return nil, fmt.Errorf("failed to get keyword at array index %v: %w", i, err) } values = append(values, *md) @@ -245,20 +245,20 @@ func NewMetadataOpcodes(publisherMetadataHandle EvtHandle) ([]MetadataOpcode, er arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) if !ok { - return nil, errors.Errorf("unexpected handle type: %T", v) + return nil, fmt.Errorf("unexpected handle type: %T", v) } defer arrayHandle.Close() arrayLen, err := EvtGetObjectArraySize(arrayHandle) if err != nil { - return nil, errors.Wrap(err, "failed to get opcode array length") + return nil, fmt.Errorf("failed to get opcode array length: %w", err) } var values []MetadataOpcode for i := uint32(0); i < arrayLen; i++ { md, err := NewMetadataOpcode(publisherMetadataHandle, arrayHandle, i) if err != nil { - return nil, errors.Wrapf(err, "failed to get opcode at array index %v", i) + return nil, fmt.Errorf("failed to get opcode at array index %v: %w", i, err) } values = append(values, *md) @@ -318,20 +318,20 @@ func NewMetadataLevels(publisherMetadataHandle EvtHandle) ([]MetadataLevel, erro arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) if !ok { - return nil, errors.Errorf("unexpected handle type: %T", v) + return nil, fmt.Errorf("unexpected handle type: %T", v) } defer arrayHandle.Close() arrayLen, err := EvtGetObjectArraySize(arrayHandle) if err != nil { - return nil, errors.Wrap(err, "failed to get level array length") + return nil, fmt.Errorf("failed to get level array length: %w", err) } var values []MetadataLevel for i := uint32(0); i < arrayLen; i++ { md, err := NewMetadataLevel(publisherMetadataHandle, arrayHandle, i) if err != nil { - return nil, errors.Wrapf(err, "failed to get level at array index %v", i) + return nil, fmt.Errorf("failed to get level at array index %v: %w", i, err) } values = append(values, *md) @@ -392,20 +392,20 @@ func NewMetadataTasks(publisherMetadataHandle EvtHandle) ([]MetadataTask, error) arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) if !ok { - return nil, errors.Errorf("unexpected handle type: %T", v) + return nil, fmt.Errorf("unexpected handle type: %T", v) } defer arrayHandle.Close() arrayLen, err := EvtGetObjectArraySize(arrayHandle) if err != nil { - return nil, errors.Wrap(err, "failed to get task array length") + return nil, fmt.Errorf("failed to get task array length: %w", err) } var values []MetadataTask for i := uint32(0); i < arrayLen; i++ { md, err := NewMetadataTask(publisherMetadataHandle, arrayHandle, i) if err != nil { - return nil, errors.Wrapf(err, "failed to get task at array index %v", i) + return nil, fmt.Errorf("failed to get task at array index %v: %w", i, err) } values = append(values, *md) @@ -473,20 +473,20 @@ func NewMetadataChannels(publisherMetadataHandle EvtHandle) ([]MetadataChannel, arrayHandle, ok := v.(EvtObjectArrayPropertyHandle) if !ok { - return nil, errors.Errorf("unexpected handle type: %T", v) + return nil, fmt.Errorf("unexpected handle type: %T", v) } defer arrayHandle.Close() arrayLen, err := EvtGetObjectArraySize(arrayHandle) if err != nil { - return nil, errors.Wrap(err, "failed to get task array length") + return nil, fmt.Errorf("failed to get task array length: %w", err) } var values []MetadataChannel for i := uint32(0); i < arrayLen; i++ { md, err := NewMetadataChannel(publisherMetadataHandle, arrayHandle, i) if err != nil { - return nil, errors.Wrapf(err, "failed to get task at array index %v", i) + return nil, fmt.Errorf("failed to get task at array index %v: %w", i, err) } values = append(values, *md) @@ -548,8 +548,8 @@ type EventMetadataIterator struct { func NewEventMetadataIterator(publisher *PublisherMetadata) (*EventMetadataIterator, error) { eventMetadataEnumHandle, err := _EvtOpenEventMetadataEnum(publisher.Handle, 0) - if err != nil { - return nil, errors.Wrap(err, "failed to open event metadata enumerator with EvtOpenEventMetadataEnum") + if err != nil && err != windows.ERROR_FILE_NOT_FOUND { //nolint:errorlint // Bad linter! This is always errno or nil. + return nil, fmt.Errorf("failed to open event metadata enumerator with EvtOpenEventMetadataEnum: %w (%#v)", err, err) } return &EventMetadataIterator{ @@ -569,14 +569,18 @@ func (itr *EventMetadataIterator) Close() error { // no more items or an error occurred. You should call Err() to check for an // error. func (itr *EventMetadataIterator) Next() bool { + if itr.eventMetadataEnumHandle == 0 { + // This is only the case when we could not find the event metadata file. + return false + } // Close existing handle. itr.currentEvent.Close() var err error itr.currentEvent, err = _EvtNextEventMetadata(itr.eventMetadataEnumHandle, 0) if err != nil { - if err != windows.ERROR_NO_MORE_ITEMS { - itr.lastErr = errors.Wrap(err, "failed advancing to next event metadata handle") + if err != windows.ERROR_NO_MORE_ITEMS { //nolint:errorlint // Bad linter! This is always errno or nil. + itr.lastErr = fmt.Errorf("failed advancing to next event metadata handle: %w", err) } return false } @@ -589,7 +593,7 @@ func (itr *EventMetadataIterator) Err() error { } func typeCastError(expected, got interface{}) error { - return errors.Errorf("wrong type for property. expected:%T got:%T", expected, got) + return fmt.Errorf("wrong type for property. expected:%T got:%T", expected, got) } func (itr *EventMetadataIterator) uint32Property(propertyID EvtEventMetadataPropertyID) (uint32, error) { diff --git a/winlogbeat/sys/wineventlog/renderer.go b/winlogbeat/sys/wineventlog/renderer.go index a4aba2d45fe..cc0adfd930a 100644 --- a/winlogbeat/sys/wineventlog/renderer.go +++ b/winlogbeat/sys/wineventlog/renderer.go @@ -186,7 +186,6 @@ func (r *Renderer) renderSystem(handle EvtHandle, event *winevent.Event) error { continue } - //nolint:errcheck // Bad linter! switch property { case EvtSystemProviderName: event.Provider.Name = data.(string) diff --git a/winlogbeat/sys/wineventlog/syscall_windows.go b/winlogbeat/sys/wineventlog/syscall_windows.go index bc895ed9c25..2d332e49d01 100644 --- a/winlogbeat/sys/wineventlog/syscall_windows.go +++ b/winlogbeat/sys/wineventlog/syscall_windows.go @@ -23,7 +23,6 @@ import ( "time" "unsafe" - "github.com/pkg/errors" "golang.org/x/sys/windows" "github.com/elastic/beats/v7/winlogbeat/sys" @@ -237,7 +236,7 @@ const ( EVENTLOG_VERBOSE_LEVEL ) -// Mapping of event levels to their string representations. +// EventLevelToString maps event levels to their string representations. var EventLevelToString = map[EventLevel]string{ EVENTLOG_LOGALWAYS_LEVEL: "Information", EVENTLOG_INFORMATION_LEVEL: "Information", @@ -492,7 +491,7 @@ func (v EvtVariant) Data(buf []byte) (interface{}, error) { case EvtVarTypeEvtHandle: return EvtHandle(v.ValueAsUintPtr()), nil default: - return nil, errors.Errorf("unhandled type: %d", typ) + return nil, fmt.Errorf("unhandled type: %d", typ) } } @@ -547,15 +546,15 @@ const ( func EvtGetPublisherMetadataProperty(publisherMetadataHandle EvtHandle, propertyID EvtPublisherMetadataPropertyID) (interface{}, error) { var bufferUsed uint32 err := _EvtGetPublisherMetadataProperty(publisherMetadataHandle, propertyID, 0, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { - return "", errors.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %v", err) + if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // Bad linter! This is always errno or nil. + return "", fmt.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %w (%#v)", err, err) } buf := make([]byte, bufferUsed) pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) err = _EvtGetPublisherMetadataProperty(publisherMetadataHandle, propertyID, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) if err != nil { - return nil, errors.Wrap(err, "failed in EvtGetPublisherMetadataProperty") + return nil, fmt.Errorf("failed in EvtGetPublisherMetadataProperty: %w", err) } v, err := pEvtVariant.Data(buf) @@ -574,20 +573,20 @@ func EvtGetPublisherMetadataProperty(publisherMetadataHandle EvtHandle, property func EvtGetObjectArrayProperty(arrayHandle EvtObjectArrayPropertyHandle, propertyID EvtPublisherMetadataPropertyID, index uint32) (interface{}, error) { var bufferUsed uint32 err := _EvtGetObjectArrayProperty(arrayHandle, propertyID, index, 0, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { - return nil, errors.Wrap(err, "failed in EvtGetObjectArrayProperty, expected ERROR_INSUFFICIENT_BUFFER") + if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // Bad linter! This is always errno or nil. + return nil, fmt.Errorf("failed in EvtGetObjectArrayProperty, expected ERROR_INSUFFICIENT_BUFFER: %w", err) } buf := make([]byte, bufferUsed) pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) err = _EvtGetObjectArrayProperty(arrayHandle, propertyID, index, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) if err != nil { - return nil, errors.Wrap(err, "failed in EvtGetObjectArrayProperty") + return nil, fmt.Errorf("failed in EvtGetObjectArrayProperty: %w", err) } value, err := pEvtVariant.Data(buf) if err != nil { - return nil, errors.Wrap(err, "failed to read EVT_VARIANT value") + return nil, fmt.Errorf("failed to read EVT_VARIANT value: %w", err) } return value, nil } @@ -609,15 +608,15 @@ func EvtGetObjectArraySize(handle EvtObjectArrayPropertyHandle) (uint32, error) func GetEventMetadataProperty(metadataHandle EvtHandle, propertyID EvtEventMetadataPropertyID) (interface{}, error) { var bufferUsed uint32 err := _EvtGetEventMetadataProperty(metadataHandle, 8, 0, 0, nil, &bufferUsed) - if err != windows.ERROR_INSUFFICIENT_BUFFER { - return nil, errors.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %v", err) + if err != windows.ERROR_INSUFFICIENT_BUFFER { //nolint:errorlint // Bad linter! This is always errno or nil. + return nil, fmt.Errorf("expected ERROR_INSUFFICIENT_BUFFER but got %w (%#v)", err, err) } buf := make([]byte, bufferUsed) pEvtVariant := (*EvtVariant)(unsafe.Pointer(&buf[0])) err = _EvtGetEventMetadataProperty(metadataHandle, propertyID, 0, uint32(len(buf)), pEvtVariant, &bufferUsed) if err != nil { - return nil, errors.Wrap(err, "_EvtGetEventMetadataProperty") + return nil, fmt.Errorf("_EvtGetEventMetadataProperty: %w", err) } return pEvtVariant.Data(buf) diff --git a/winlogbeat/sys/wineventlog/template.go b/winlogbeat/sys/wineventlog/template.go index ac19322ef0a..9ac34062f99 100644 --- a/winlogbeat/sys/wineventlog/template.go +++ b/winlogbeat/sys/wineventlog/template.go @@ -17,9 +17,7 @@ package wineventlog -import ( - "encoding/xml" -) +import "encoding/xml" type eventTemplate struct { Data []EventData `xml:"data"` diff --git a/winlogbeat/sys/wineventlog/wineventlog_windows.go b/winlogbeat/sys/wineventlog/wineventlog_windows.go index 96d4187387d..ffa7a2ae150 100644 --- a/winlogbeat/sys/wineventlog/wineventlog_windows.go +++ b/winlogbeat/sys/wineventlog/wineventlog_windows.go @@ -559,7 +559,6 @@ func renderXML(eventHandle EvtHandle, flag EvtRenderFlag, renderBuf []byte, out } if int(bufferUsed) > len(renderBuf) { - //nolint:stylecheck // These are proper nouns. return fmt.Errorf("Windows EvtRender reported that wrote %d bytes "+ "to the buffer, but the buffer can only hold %d bytes", bufferUsed, len(renderBuf)) diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index dc86c9ead19..01d2a0298d3 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/auditbeat check; - make -C x-pack/auditbeat update; - make -C auditbeat check; - make -C auditbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/auditbeat/module/system/host/_meta/data.json b/x-pack/auditbeat/module/system/host/_meta/data.json index a4494027c6b..a2b0bef9460 100644 --- a/x-pack/auditbeat/module/system/host/_meta/data.json +++ b/x-pack/auditbeat/module/system/host/_meta/data.json @@ -33,10 +33,10 @@ "fe80::42:9eff:fed3:d888" ], "mac": [ - "02:2d:fd:81:e7:47", - "08:00:27:1f:71:60", - "02:42:83:be:1a:3a", - "02:42:9e:d3:d8:88" + "02-2D-FD-81-E7-47", + "08-00-27-1F-71-60", + "02-42-83-BE-1A-3A", + "02-42-9E-D3-D8-88" ], "os": { "family": "debian", diff --git a/x-pack/auditbeat/module/system/host/host.go b/x-pack/auditbeat/module/system/host/host.go index c2a983ca3c9..37860fcc436 100644 --- a/x-pack/auditbeat/module/system/host/host.go +++ b/x-pack/auditbeat/module/system/host/host.go @@ -95,6 +95,7 @@ type Host struct { // changeDetectionHash creates a hash of selected parts of the host information. // This is used later to detect changes to a host over time. +//nolint:errcheck // All checks are for writes to a hasher. func (host *Host) changeDetectionHash() uint64 { h := xxhash.New() @@ -114,6 +115,7 @@ func (host *Host) changeDetectionHash() uint64 { return h.Sum64() } +//nolint:errcheck // All checks are for mapstr.Put. func (host *Host) toMapStr() mapstr.M { mapstr := mapstr.M{ // https://github.com/elastic/ecs#-host-fields @@ -155,9 +157,8 @@ func (host *Host) toMapStr() mapstr.M { var macStrings []string for _, mac := range host.Macs { - macStr := mac.String() - if macStr != "" { - macStrings = append(macStrings, macStr) + if len(mac) != 0 { + macStrings = append(macStrings, formatHardwareAddr(mac)) } } mapstr.Put("mac", macStrings) @@ -165,6 +166,19 @@ func (host *Host) toMapStr() mapstr.M { return mapstr } +// formatHardwareAddr formats hardware addresses according to the ECS spec. +func formatHardwareAddr(addr net.HardwareAddr) string { + buf := make([]byte, 0, len(addr)*3-1) + for _, b := range addr { + if len(buf) != 0 { + buf = append(buf, '-') + } + const hexDigit = "0123456789ABCDEF" + buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf]) + } + return string(buf) +} + func init() { mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, mb.DefaultMetricSet(), @@ -273,6 +287,7 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { var events []mb.Event // Report ID changes as a separate, special event. + //nolint:errcheck // All checks are for mapstr.Put. if ms.lastHost.Info.UniqueID != currentHost.Info.UniqueID { /* Issue two events - one for the host with the old ID, one for the new @@ -309,7 +324,7 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { } if len(events) > 0 { - ms.saveStateToDisk() + return ms.saveStateToDisk() } return nil @@ -336,6 +351,7 @@ func getHost() (*Host, error) { return host, nil } +//nolint:errcheck // All checks are for mapstr.CopyFieldsTo. func hostEvent(host *Host, eventType string, action eventAction) mb.Event { hostFields := host.toMapStr() @@ -460,9 +476,11 @@ func (ms *MetricSet) restoreStateFromDisk() error { if decoder != nil { var lastHost Host err = decoder.Decode(&lastHost) - if err == nil { + switch err { //nolint:errorlint // Bad linter! io.EOF is never wrapped. + case nil: ms.lastHost = &lastHost - } else if err != io.EOF { + case io.EOF: + default: return fmt.Errorf("error decoding host information: %w", err) } } diff --git a/x-pack/auditbeat/module/system/socket/helper/linkedlist.go b/x-pack/auditbeat/module/system/socket/helper/linkedlist.go index b7e91745c4d..3daf2351b12 100644 --- a/x-pack/auditbeat/module/system/socket/helper/linkedlist.go +++ b/x-pack/auditbeat/module/system/socket/helper/linkedlist.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build (linux && 386) || (linux && amd64) +// +build linux,386 linux,amd64 + package helper import "time" diff --git a/x-pack/auditbeat/module/system/socket/helper/linkedlist_test.go b/x-pack/auditbeat/module/system/socket/helper/linkedlist_test.go index d620e899edc..6902b87755e 100644 --- a/x-pack/auditbeat/module/system/socket/helper/linkedlist_test.go +++ b/x-pack/auditbeat/module/system/socket/helper/linkedlist_test.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build (linux && 386) || (linux && amd64) +// +build linux,386 linux,amd64 + package helper import ( diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index d639c257cbb..7383bedac7d 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/filebeat check; - make -C x-pack/filebeat update; - make -C filebeat check; - make -C filebeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index e181c8abeb2..8822df97903 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -224,6 +224,7 @@ Some built-in helper functions are provided to work with the input state inside - `replaceAll(old, new, s)`: replaces all non-overlapping instances of `old` with `new` in `s`. Example: `[[ replaceAll "some" "my" "some value" ]]` returns `my value`. - `sprintf`: formats according to a format specifier and returns the resulting string. Refer to https://pkg.go.dev/fmt#Sprintf[the Go docs] for usage. Example: `[[sprintf "%d:%q" 34 "quote this"]]` - `toInt`: converts a value of any type to an integer when possible. Returns 0 if the conversion fails. +- `toJSON`: converts a value to a JSON string. This can be used with `value_type: json` to create an object from a template. Example: `[[ toJSON .last_response.body.pagingIdentifiers ]]`. - `urlEncode`: URL encodes the supplied string. Example `[[urlEncode "string1"]]`. Example `[[urlEncode ""]]` will return `%3Cstring1%3E`. - `userAgent`: generates the User Agent with optional additional values. If no arguments are provided, it will generate the default User Agent that is added to all requests by default. It is recommended to delete the existing User-Agent header before setting a new one. Example: `[[ userAgent "integration/1.2.3" ]]` would generate `Elastic-Filebeat/8.1.0 (darwin; amd64; 9b893e88cfe109e64638d65c58fd75c2ff695402; 2021-12-15 13:20:00 +0000 UTC; integration_name/1.2.3)` - `uuid`: returns a random UUID such as `a11e8780-e3e7-46d0-8e76-f66e75acf019`. Example: `[[ uuid ]]` @@ -546,6 +547,43 @@ filebeat.inputs: value: '[[now (parseDuration "-1h")]]' ---- +[float] +==== `request.tracer.filename` + +It is possible to log httpjson requests and responses to a local file-system for debugging configurations. +This option is enabled by setting the `request.tracer.filename` value. Additional options are available to +tune log rotation behavior. + +Enabling this option compromises security and should only be used for debugging. + +[float] +==== `request.tracer.maxsize` + +This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default +logs are allowed to reach 1MB before rotation. + +[float] +==== `request.tracer.maxage` + +This specifies the number days to retain rotated log files. If it is not set, log files are retained +indefinitely. + +[float] +==== `request.tracer.maxbackups` + +The number of old logs to retain. If it is not set all old logs are retained subject to the `request.tracer.maxage` +setting. + +[float] +==== `request.tracer.localtime` + +Whether to use the host's local time rather that UTC for timestamping rotated log file names. + +[float] +==== `request.tracer.compress` + +This determines whether rotated logs should be gzip compressed. + [float] ==== `response.decode_as` diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 91bf5fee773..c1382ae3627 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -13,6 +13,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cometd" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/aws" diff --git a/x-pack/filebeat/input/awscloudwatch/cloudwatch.go b/x-pack/filebeat/input/awscloudwatch/cloudwatch.go index 4639d55905e..ad522806c28 100644 --- a/x-pack/filebeat/input/awscloudwatch/cloudwatch.go +++ b/x-pack/filebeat/input/awscloudwatch/cloudwatch.go @@ -61,7 +61,7 @@ func (p *cloudwatchPoller) run(svc *cloudwatchlogs.Client, logGroup string, star err := p.getLogEventsFromCloudWatch(svc, logGroup, startTime, endTime, logProcessor) if err != nil { var errRequestCanceled *awssdk.RequestCanceledError - if errors.As(errRequestCanceled, &err) { + if errors.As(err, &errRequestCanceled) { p.log.Error("getLogEventsFromCloudWatch failed with RequestCanceledError: ", err) } p.log.Error("getLogEventsFromCloudWatch failed: ", err) diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index c116af21bc9..9269dfe96db 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -313,6 +313,7 @@ func getProviderFromDomain(endpoint string, ProviderOverride string) string { "c2s.ic.gov": "aws", "amazonaws.com.cn": "aws", "backblazeb2.com": "backblaze", + "cloudflarestorage.com": "cloudflare", "wasabisys.com": "wasabi", "digitaloceanspaces.com": "digitalocean", "dream.io": "dreamhost", diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index a5086261a98..6fc3da78d46 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -18,6 +18,7 @@ import ( "path" "path/filepath" "runtime" + "strings" "testing" "time" @@ -88,7 +89,6 @@ file_selectors: - regex: 'events-array.json$' expand_event_list_from_field: Events - content_type: application/json include_s3_metadata: - last-modified - x-amz-version-id @@ -97,7 +97,6 @@ file_selectors: - Content-Type - regex: '\.(?:nd)?json(\.gz)?$' - content_type: application/json - regex: 'multiline.txt$' parsers: @@ -117,7 +116,6 @@ file_selectors: - regex: 'events-array.json$' expand_event_list_from_field: Events - content_type: application/json include_s3_metadata: - last-modified - x-amz-version-id @@ -126,7 +124,6 @@ file_selectors: - Content-Type - regex: '\.(?:nd)?json(\.gz)?$' - content_type: application/json - regex: 'multiline.txt$' parsers: @@ -328,11 +325,19 @@ func uploadS3TestFiles(t *testing.T, region, bucket string, filenames ...string) t.Fatalf("Failed to open file %q, %v", filename, err) } + contentType := "" + if strings.HasSuffix(filename, "ndjson") || strings.HasSuffix(filename, "ndjson.gz") { + contentType = contentTypeNDJSON + "; charset=UTF-8" + } else if strings.HasSuffix(filename, "json") || strings.HasSuffix(filename, "json.gz") { + contentType = contentTypeJSON + "; charset=UTF-8" + } + // Upload the file to S3. result, err := uploader.Upload(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(filepath.Base(filename)), - Body: bytes.NewReader(data), + Bucket: aws.String(bucket), + Key: aws.String(filepath.Base(filename)), + Body: bytes.NewReader(data), + ContentType: aws.String(contentType), }) if err != nil { t.Fatalf("Failed to upload file %q: %v", filename, err) diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index 1a10eb0115f..953e8388fc0 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -20,6 +20,7 @@ func TestGetProviderFromDomain(t *testing.T) { assert.Equal(t, "aws", getProviderFromDomain("c2s.ic.gov", "")) assert.Equal(t, "aws", getProviderFromDomain("amazonaws.com.cn", "")) assert.Equal(t, "backblaze", getProviderFromDomain("https://backblazeb2.com", "")) + assert.Equal(t, "cloudflare", getProviderFromDomain("https://1234567890.r2.cloudflarestorage.com", "")) assert.Equal(t, "wasabi", getProviderFromDomain("https://wasabisys.com", "")) assert.Equal(t, "digitalocean", getProviderFromDomain("https://digitaloceanspaces.com", "")) assert.Equal(t, "dreamhost", getProviderFromDomain("https://dream.io", "")) diff --git a/x-pack/filebeat/input/awss3/s3_objects.go b/x-pack/filebeat/input/awss3/s3_objects.go index 826b65f1aca..a1d70c604c2 100644 --- a/x-pack/filebeat/input/awss3/s3_objects.go +++ b/x-pack/filebeat/input/awss3/s3_objects.go @@ -153,7 +153,7 @@ func (p *s3ObjectProcessor) ProcessS3Object() error { // Process object content stream. switch { - case contentType == contentTypeJSON || contentType == contentTypeNDJSON: + case strings.HasPrefix(contentType, contentTypeJSON) || strings.HasPrefix(contentType, contentTypeNDJSON): err = p.readJSON(reader) default: err = p.readFile(reader) diff --git a/x-pack/filebeat/input/default-inputs/inputs_aix.go b/x-pack/filebeat/input/default-inputs/inputs_aix.go index a1cdf5da43e..f46d8ed1f25 100644 --- a/x-pack/filebeat/input/default-inputs/inputs_aix.go +++ b/x-pack/filebeat/input/default-inputs/inputs_aix.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" + "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" "github.com/elastic/elastic-agent-libs/logp" ) @@ -21,5 +22,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 httpjson.Plugin(log, store), o365audit.Plugin(log, store), awss3.Plugin(store), + lumberjack.Plugin(), } } diff --git a/x-pack/filebeat/input/default-inputs/inputs_other.go b/x-pack/filebeat/input/default-inputs/inputs_other.go index b100612d58c..b87faaed46a 100644 --- a/x-pack/filebeat/input/default-inputs/inputs_other.go +++ b/x-pack/filebeat/input/default-inputs/inputs_other.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" + "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" "github.com/elastic/elastic-agent-libs/logp" ) @@ -28,5 +29,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 o365audit.Plugin(log, store), awss3.Plugin(store), awscloudwatch.Plugin(), + lumberjack.Plugin(), } } diff --git a/x-pack/filebeat/input/gcppubsub/_meta/Dockerfile b/x-pack/filebeat/input/gcppubsub/_meta/Dockerfile index eea35d42374..9fa271e8838 100644 --- a/x-pack/filebeat/input/gcppubsub/_meta/Dockerfile +++ b/x-pack/filebeat/input/gcppubsub/_meta/Dockerfile @@ -28,6 +28,6 @@ RUN \ RUN \ mkdir /data -HEALTHCHECK --interval=1s --retries=90 CMD curl -s -f http://localhost:8432/ +HEALTHCHECK --interval=1s --retries=90 CMD curl -s -f --http2 http://localhost:8432/ CMD gcloud beta emulators pubsub start --data-dir /data --host-port "0.0.0.0:8432" diff --git a/x-pack/filebeat/input/gcppubsub/_meta/supported-versions.yml b/x-pack/filebeat/input/gcppubsub/_meta/supported-versions.yml index ac4ba96e699..afc1148bd83 100644 --- a/x-pack/filebeat/input/gcppubsub/_meta/supported-versions.yml +++ b/x-pack/filebeat/input/gcppubsub/_meta/supported-versions.yml @@ -1,2 +1,2 @@ variants: - - SDK_VERSION: 293.0.0-0 + - SDK_VERSION: 398.0.0-0 diff --git a/x-pack/filebeat/input/gcppubsub/docker-compose.yml b/x-pack/filebeat/input/gcppubsub/docker-compose.yml index 6afb855b8a4..60ce3ebd83f 100644 --- a/x-pack/filebeat/input/gcppubsub/docker-compose.yml +++ b/x-pack/filebeat/input/gcppubsub/docker-compose.yml @@ -2,10 +2,10 @@ version: '2.3' services: googlepubsub: - image: docker.elastic.co/integrations-ci/beats-googlepubsub:emulator-${SDK_VERSION:-293.0.0-0}-1 + image: docker.elastic.co/integrations-ci/beats-googlepubsub:emulator-${SDK_VERSION:-398.0.0-0}-1 build: context: ./_meta args: - SDK_VERSION: ${SDK_VERSION:-293.0.0-0} + SDK_VERSION: ${SDK_VERSION:-398.0.0-0} ports: - - 8432 + - '127.0.0.1:8432:8432' diff --git a/x-pack/filebeat/input/gcppubsub/input.go b/x-pack/filebeat/input/gcppubsub/input.go index 11a6a96ed2b..512c7dc561b 100644 --- a/x-pack/filebeat/input/gcppubsub/input.go +++ b/x-pack/filebeat/input/gcppubsub/input.go @@ -8,14 +8,16 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "sync" "time" "cloud.google.com/go/pubsub" - "github.com/pkg/errors" + "golang.org/x/time/rate" "google.golang.org/api/option" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" @@ -32,17 +34,20 @@ import ( const ( inputName = "gcp-pubsub" oldInputName = "google-pubsub" + + // retryInterval is the minimum duration between pub/sub client retries. + retryInterval = 30 * time.Second ) func init() { err := input.Register(inputName, NewInput) if err != nil { - panic(errors.Wrapf(err, "failed to register %v input", inputName)) + panic(fmt.Errorf("failed to register %v input: %w", inputName, err)) } err = input.Register(oldInputName, NewInput) if err != nil { - panic(errors.Wrapf(err, "failed to register %v input", oldInputName)) + panic(fmt.Errorf("failed to register %v input: %w", oldInputName, err)) } } @@ -139,9 +144,28 @@ func (in *pubsubInput) Run() { defer in.log.Info("Pub/Sub input worker has stopped.") defer in.workerWg.Done() defer in.workerCancel() - if err := in.run(); err != nil { - in.log.Error(err) - return + + // Throttle pubsub client restarts. + rt := rate.NewLimiter(rate.Every(retryInterval), 1) + + // Watchdog to keep the worker operating after an error. + for in.workerCtx.Err() == nil { + // Rate limit. + if err := rt.Wait(in.workerCtx); err != nil { + continue + } + + if err := in.run(); err != nil { + if in.workerCtx.Err() == nil { + in.log.Warnw("Restarting failed Pub/Sub input worker.", "error", err) + continue + } + + // Log any non-cancellation error before stopping. + if !errors.Is(err, context.Canceled) { + in.log.Errorw("Pub/Sub input worker failed.", "error", err) + } + } } }() }) @@ -160,7 +184,7 @@ func (in *pubsubInput) run() error { // Setup our subscription to the topic. sub, err := in.getOrCreateSubscription(ctx, client) if err != nil { - return errors.Wrap(err, "failed to subscribe to pub/sub topic") + return fmt.Errorf("failed to subscribe to pub/sub topic: %w", err) } sub.ReceiveSettings.NumGoroutines = in.Subscription.NumGoroutines sub.ReceiveSettings.MaxOutstandingMessages = in.Subscription.MaxOutstandingMessages @@ -215,7 +239,7 @@ func makeEvent(topicID string, msg *pubsub.Message) beat.Event { event.SetID(id) if len(msg.Attributes) > 0 { - event.PutValue("labels", msg.Attributes) + event.Fields["labels"] = msg.Attributes } return event @@ -226,7 +250,7 @@ func (in *pubsubInput) getOrCreateSubscription(ctx context.Context, client *pubs exists, err := sub.Exists(ctx) if err != nil { - return nil, errors.Wrap(err, "failed to check if subscription exists") + return nil, fmt.Errorf("failed to check if subscription exists: %w", err) } if exists { return sub, nil @@ -238,7 +262,7 @@ func (in *pubsubInput) getOrCreateSubscription(ctx context.Context, client *pubs Topic: client.Topic(in.Topic), }) if err != nil { - return nil, errors.Wrap(err, "failed to create subscription") + return nil, fmt.Errorf("failed to create subscription: %w", err) } in.log.Debug("Created new subscription.") return sub, nil @@ -251,8 +275,8 @@ func (in *pubsubInput) newPubsubClient(ctx context.Context) (*pubsub.Client, err opts := []option.ClientOption{option.WithUserAgent(useragent.UserAgent("Filebeat", version.GetDefaultVersion(), version.Commit(), version.BuildTime().String()))} if in.AlternativeHost != "" { - // this will be typically set because we want to point the input to a testing pubsub emulator - conn, err := grpc.Dial(in.AlternativeHost, grpc.WithInsecure()) + // This will be typically set because we want to point the input to a testing pubsub emulator. + conn, err := grpc.Dial(in.AlternativeHost, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, fmt.Errorf("cannot connect to alternative host %q: %w", in.AlternativeHost, err) } diff --git a/x-pack/filebeat/input/httpjson/chain.go b/x-pack/filebeat/input/httpjson/chain.go index 14fcafb9ce2..ad61c991dd6 100644 --- a/x-pack/filebeat/input/httpjson/chain.go +++ b/x-pack/filebeat/input/httpjson/chain.go @@ -69,10 +69,10 @@ type chainConfig struct { // will contain replace string with original URL to make a skeleton for the // call request. type stepConfig struct { - Auth *authConfig `config:"auth"` - Request requestConfig `config:"request" validate:"required"` - Response responseChainConfig `config:"response,omitempty"` - Replace string `config:"replace,omitempty"` + Auth *authConfig `config:"auth"` + Request *requestConfig `config:"request" validate:"required"` + Response *responseChainConfig `config:"response,omitempty"` + Replace string `config:"replace,omitempty"` } // whileConfig will contain basic properties like auth parameters, request parameters, @@ -82,11 +82,11 @@ type stepConfig struct { // expression is evaluated to "true" or request.retry.max_attempts is exhausted. If // request.retry.max_attempts is not specified , the max_attempts is always 1. type whileConfig struct { - Auth *authConfig `config:"auth"` - Request requestConfig `config:"request" validate:"required"` - Response responseChainConfig `config:"response,omitempty"` - Replace string `config:"replace,omitempty"` - Until *valueTpl `config:"until" validate:"required"` + Auth *authConfig `config:"auth"` + Request *requestConfig `config:"request" validate:"required"` + Response *responseChainConfig `config:"response,omitempty"` + Replace string `config:"replace,omitempty"` + Until *valueTpl `config:"until" validate:"required"` } type responseChainConfig struct { @@ -100,13 +100,13 @@ func defaultChainConfig() config { { While: &whileConfig{ Auth: chaincfg.Auth, - Request: *chaincfg.Request, - Response: responseChainConfig{}, + Request: chaincfg.Request, + Response: &responseChainConfig{}, }, Step: &stepConfig{ Auth: chaincfg.Auth, - Request: *chaincfg.Request, - Response: responseChainConfig{}, + Request: chaincfg.Request, + Response: &responseChainConfig{}, }, }, } diff --git a/x-pack/filebeat/input/httpjson/config_request.go b/x-pack/filebeat/input/httpjson/config_request.go index 2e2b2402f9f..dcfda22ee1d 100644 --- a/x-pack/filebeat/input/httpjson/config_request.go +++ b/x-pack/filebeat/input/httpjson/config_request.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "gopkg.in/natefinch/lumberjack.v2" + "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/transport/httpcommon" ) @@ -100,6 +102,8 @@ type requestConfig struct { Transforms transformsConfig `config:"transforms"` Transport httpcommon.HTTPTransportSettings `config:",inline"` + + Tracer *lumberjack.Logger `config:"tracer"` } func (c *requestConfig) Validate() error { @@ -124,5 +128,17 @@ func (c *requestConfig) Validate() error { } } + if c.Tracer != nil { + if c.Tracer.Filename == "" { + return errors.New("request tracer must have a filename if used") + } + if c.Tracer.MaxSize == 0 { + // By default Lumberjack caps file sizes at 100MB which + // is excessive for a debugging logger, so default to 1MB + // which is the minimum. + c.Tracer.MaxSize = 1 + } + } + return nil } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 84241a800a3..87ea2a2257d 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -14,13 +14,16 @@ import ( "time" retryablehttp "github.com/hashicorp/go-retryablehttp" + "go.elastic.co/ecszap" "go.uber.org/zap" + "go.uber.org/zap/zapcore" v2 "github.com/elastic/beats/v7/filebeat/input/v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/httplog" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/transport/httpcommon" @@ -160,6 +163,18 @@ func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpC return nil, err } + if config.Request.Tracer != nil { + w := zapcore.AddSync(config.Request.Tracer) + core := ecszap.NewCore( + ecszap.NewDefaultEncoderConfig(), + w, + zap.DebugLevel, + ) + traceLogger := zap.New(core) + + netHTTPClient.Transport = httplog.NewLoggingRoundTripper(netHTTPClient.Transport, traceLogger) + } + netHTTPClient.CheckRedirect = checkRedirect(config.Request, log) client := &retryablehttp.Client{ diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 69617d82a5b..0ec01ce968f 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -578,6 +578,36 @@ func TestInput(t *testing.T) { `{"space":{"cake":"pumpkin"}}`, }, }, + { + name: "Test pagination when used with chaining", + setupServer: newChainPaginationTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": http.MethodGet, + "response.pagination": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.value", + "value": "[[.last_response.body.nextLink]]", + "fail_on_template_error": true, + }, + }, + }, + "chain": []interface{}{ + map[string]interface{}{ + "step": map[string]interface{}{ + "request.method": http.MethodGet, + "replace": "$.records[:].id", + }, + }, + }, + }, + handler: defaultHandler(http.MethodGet, ""), + expected: []string{ + `{"hello":{"world":"moon"}}`, + `{"space":{"cake":"pumpkin"}}`, + }, + }, } for _, testCase := range testCases { @@ -668,6 +698,34 @@ func newChainTestServer( } } +func newChainPaginationTestServer( + newServer func(http.Handler) *httptest.Server, +) func(*testing.T, http.HandlerFunc, map[string]interface{}) { + return func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerPaginationTransforms() + var serverURL string + r := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/": + link := serverURL + "/link2" + value := fmt.Sprintf(`{"records":[{"id":1}], "nextLink":"%s"}`, link) + fmt.Fprintln(w, value) + case "/1": + fmt.Fprintln(w, `{"hello":{"world":"moon"}}`) + case "/link2": + fmt.Fprintln(w, `{"records":[{"id":2}]}`) + case "/2": + fmt.Fprintln(w, `{"space":{"cake":"pumpkin"}}`) + } + }) + server := httptest.NewServer(r) + config["request.url"] = server.URL + serverURL = server.URL + config["chain.0.step.request.url"] = server.URL + "/$.records[:].id" + t.Cleanup(func() { registeredTransforms = newRegistry() }) + } +} + func newV2Context() (v2.Context, func()) { ctx, cancel := context.WithCancel(context.Background()) return v2.Context{ diff --git a/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go b/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go new file mode 100644 index 00000000000..319ec8b65eb --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/httplog/roundtripper.go @@ -0,0 +1,208 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package httplog provides http request and response transaction logging. +package httplog + +import ( + "bytes" + "encoding/base32" + "encoding/binary" + "errors" + "fmt" + "io" + "net/http" + "net/http/httputil" + "strconv" + "time" + + "go.uber.org/atomic" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var _ http.RoundTripper = (*LoggingRoundTripper)(nil) + +// TraceIDKey is key used to add a trace.id value to the context of HTTP +// requests. The value will be logged by LoggingRoundTripper. +const TraceIDKey = contextKey("trace.id") + +type contextKey string + +// NewLoggingRoundTripper returns a LoggingRoundTripper that logs requests and +// responses to the provided logger. +func NewLoggingRoundTripper(next http.RoundTripper, logger *zap.Logger) *LoggingRoundTripper { + return &LoggingRoundTripper{ + transport: next, + logger: logger, + txBaseID: newID(), + txIDCounter: atomic.NewUint64(0), + } +} + +// LoggingRoundTripper is an http.RoundTripper that logs requests and responses. +type LoggingRoundTripper struct { + transport http.RoundTripper + logger *zap.Logger // Destination logger. + txBaseID string // Random value to make transaction IDs unique. + txIDCounter *atomic.Uint64 // Transaction ID counter that is incremented for each request. +} + +// RoundTrip implements the http.RoundTripper interface, logging +// the request and response to the underlying logger. +// +// Fields logged in requests: +// url.original +// url.scheme +// url.path +// url.domain +// url.port +// url.query +// http.request +// user_agent.original +// http.request.body.content +// http.request.body.bytes +// http.request.mime_type +// event.original (the full request and body from httputil.DumpRequestOut) +// +// Fields logged in responses: +// http.response.status_code +// http.response.body.content +// http.response.body.bytes +// http.response.mime_type +// event.original (the full response and body from httputil.DumpResponse) +// +func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Create a child logger for this request. + log := rt.logger.With( + zap.String("transaction.id", rt.nextTxID()), + ) + + if v := req.Context().Value(TraceIDKey); v != nil { + if traceID, ok := v.(string); ok { + log = log.With(zap.String("trace.id", traceID)) + } + } + + reqParts := []zapcore.Field{ + zap.String("url.original", req.URL.String()), + zap.String("url.scheme", req.URL.Scheme), + zap.String("url.path", req.URL.Path), + zap.String("url.domain", req.URL.Hostname()), + zap.String("url.port", req.URL.Port()), + zap.String("url.query", req.URL.RawQuery), + zap.String("http.request.method", req.Method), + zap.String("user_agent.original", req.Header.Get("User-Agent")), + } + var ( + body []byte + err error + errorsMessages []string + ) + req.Body, body, err = copyBody(req.Body) + if err != nil { + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err)) + } else { + reqParts = append(reqParts, + zap.ByteString("http.request.body.content", body), + zap.Int("http.request.body.bytes", len(body)), + zap.String("http.request.mime_type", req.Header.Get("Content-Type")), + ) + } + message, err := httputil.DumpRequestOut(req, true) + if err != nil { + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump request: %s", err)) + } else { + reqParts = append(reqParts, zap.ByteString("event.original", message)) + } + switch len(errorsMessages) { + case 0: + case 1: + reqParts = append(reqParts, zap.String("error.message", errorsMessages[0])) + default: + reqParts = append(reqParts, zap.Strings("error.message", errorsMessages)) + } + log.Debug("HTTP request", reqParts...) + + resp, err := rt.transport.RoundTrip(req) + if err != nil { + log.Debug("HTTP response error", zap.NamedError("error.message", err)) + return resp, err + } + if resp == nil { + log.Debug("HTTP response error", noResponse) + return resp, err + } + respParts := append(reqParts[:0], + zap.Int("http.response.status_code", resp.StatusCode), + ) + errorsMessages = errorsMessages[:0] + resp.Body, body, err = copyBody(resp.Body) + if err != nil { + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err)) + } else { + respParts = append(respParts, + zap.ByteString("http.response.body.content", body), + zap.Int("http.response.body.bytes", len(body)), + zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), + ) + } + message, err = httputil.DumpResponse(resp, true) + if err != nil { + errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump response: %s", err)) + } else { + respParts = append(respParts, zap.ByteString("event.original", message)) + } + switch len(errorsMessages) { + case 0: + case 1: + respParts = append(reqParts, zap.String("error.message", errorsMessages[0])) + default: + respParts = append(reqParts, zap.Strings("error.message", errorsMessages)) + } + log.Debug("HTTP response", respParts...) + + return resp, err +} + +// nextTxID returns the next transaction.id value. It increments the internal +// request counter. +func (rt *LoggingRoundTripper) nextTxID() string { + count := rt.txIDCounter.Inc() + return rt.txBaseID + "-" + strconv.FormatUint(count, 10) +} + +var noResponse = zap.NamedError("error.message", errors.New("unexpected nil response")) + +// newID returns an ID derived from the current time. +func newID() string { + var data [8]byte + binary.LittleEndian.PutUint64(data[:], uint64(time.Now().UnixNano())) + return base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(data[:]) +} + +// copyBody is derived from drainBody in net/http/httputil/dump.go +// +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// copyBody reads all of b to memory and then returns a +// ReadCloser yielding the same bytes, and the bytes themselves. +// +// It returns an error if the initial slurp of all bytes fails. +func copyBody(b io.ReadCloser) (r io.ReadCloser, body []byte, err error) { + if b == nil || b == http.NoBody { + // No copying needed. Preserve the magic sentinel meaning of NoBody. + return http.NoBody, nil, nil + } + var buf bytes.Buffer + if _, err = buf.ReadFrom(b); err != nil { + return nil, buf.Bytes(), err + } + if err = b.Close(); err != nil { + return nil, buf.Bytes(), err + } + return io.NopCloser(&buf), buf.Bytes(), nil +} diff --git a/x-pack/filebeat/input/httpjson/rate_limiter.go b/x-pack/filebeat/input/httpjson/rate_limiter.go index dd8c2298086..1dce2fe2648 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter.go @@ -41,10 +41,6 @@ func newRateLimiterFromConfig(config *rateLimitConfig, log *logp.Logger) *rateLi func (r *rateLimiter) execute(ctx context.Context, f func() (*http.Response, error)) (*http.Response, error) { for { resp, err := f() - if err != nil { - return nil, err - } - if err != nil { return nil, fmt.Errorf("failed to read http.response.body: %w", err) } diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 17bece981b7..0e11248f8b7 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -92,18 +92,19 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro } type requestFactory struct { - url url.URL - method string - body *mapstr.M - transforms []basicTransform - user string - password string - log *logp.Logger - encoder encoderFunc - replace string - isChain bool - until *valueTpl - chainHTTPClient *httpClient + url url.URL + method string + body *mapstr.M + transforms []basicTransform + user string + password string + log *logp.Logger + encoder encoderFunc + replace string + isChain bool + until *valueTpl + chainHTTPClient *httpClient + chainResponseProcessor *responseProcessor } func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([]*requestFactory, error) { @@ -130,7 +131,7 @@ func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([] if ch.Step != nil { ts, _ := newBasicTransformsFromConfig(ch.Step.Request.Transforms, requestNamespace, log) ch.Step.Auth = tryAssignAuth(config.Auth, ch.Step.Auth) - httpClient, err := newChainHTTPClient(ctx, ch.Step.Auth, &ch.Step.Request, log) + httpClient, err := newChainHTTPClient(ctx, ch.Step.Auth, ch.Step.Request, log) if err != nil { return nil, fmt.Errorf("failed in creating chain http client with error : %w", err) } @@ -138,22 +139,24 @@ func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([] rf.user = ch.Step.Auth.Basic.User rf.password = ch.Step.Auth.Basic.Password } + responseProcessor := newChainResponseProcessor(ch, httpClient, log) rf = &requestFactory{ - url: *ch.Step.Request.URL.URL, - method: ch.Step.Request.Method, - body: ch.Step.Request.Body, - transforms: ts, - log: log, - encoder: registeredEncoders[config.Request.EncodeAs], - replace: ch.Step.Replace, - isChain: true, - chainHTTPClient: httpClient, + url: *ch.Step.Request.URL.URL, + method: ch.Step.Request.Method, + body: ch.Step.Request.Body, + transforms: ts, + log: log, + encoder: registeredEncoders[config.Request.EncodeAs], + replace: ch.Step.Replace, + isChain: true, + chainHTTPClient: httpClient, + chainResponseProcessor: responseProcessor, } } else if ch.While != nil { ts, _ := newBasicTransformsFromConfig(ch.While.Request.Transforms, requestNamespace, log) policy := newHTTPPolicy(evaluateResponse, ch.While.Until, log) ch.While.Auth = tryAssignAuth(config.Auth, ch.While.Auth) - httpClient, err := newChainHTTPClient(ctx, ch.While.Auth, &ch.While.Request, log, policy) + httpClient, err := newChainHTTPClient(ctx, ch.While.Auth, ch.While.Request, log, policy) if err != nil { return nil, fmt.Errorf("failed in creating chain http client with error : %w", err) } @@ -161,17 +164,19 @@ func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([] rf.user = ch.While.Auth.Basic.User rf.password = ch.While.Auth.Basic.Password } + responseProcessor := newChainResponseProcessor(ch, httpClient, log) rf = &requestFactory{ - url: *ch.While.Request.URL.URL, - method: ch.While.Request.Method, - body: ch.While.Request.Body, - transforms: ts, - log: log, - encoder: registeredEncoders[config.Request.EncodeAs], - replace: ch.While.Replace, - until: ch.While.Until, - isChain: true, - chainHTTPClient: httpClient, + url: *ch.While.Request.URL.URL, + method: ch.While.Request.Method, + body: ch.While.Request.Body, + transforms: ts, + log: log, + encoder: registeredEncoders[config.Request.EncodeAs], + replace: ch.While.Replace, + until: ch.While.Until, + isChain: true, + chainHTTPClient: httpClient, + chainResponseProcessor: responseProcessor, } } rfs = append(rfs, rf) @@ -225,7 +230,8 @@ func newRequester( client *httpClient, requestFactory []*requestFactory, responseProcessor []*responseProcessor, - log *logp.Logger) *requester { + log *logp.Logger, +) *requester { return &requester{ log: log, client: client, @@ -268,6 +274,7 @@ func generateNewUrl(replacement, oldUrl, id string) (url.URL, error) { return *newUrl, nil } +//nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, publisher inputcursor.Publisher) error { var ( n int @@ -276,30 +283,52 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p urlCopy url.URL urlString string httpResp *http.Response + initialResponse []*http.Response intermediateResps []*http.Response finalResps []*http.Response + isChainExpected bool + chainIndex int ) + for i, rf := range r.requestFactories { finalResps = nil intermediateResps = nil // iterate over collected ids from last response if i == 0 { // perform and store regular call responses - httpResp, err = rf.collectResponse(stdCtx, trCtx, r) //nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. + httpResp, err = rf.collectResponse(stdCtx, trCtx, r) if err != nil { return fmt.Errorf("failed to execute rf.collectResponse: %w", err) } if len(r.requestFactories) == 1 { - finalResps = append(finalResps, httpResp) //nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. - n = r.processAndPublishEvents(stdCtx, trCtx, publisher, finalResps, true, i) + finalResps = append(finalResps, httpResp) + events := r.responseProcessors[i].startProcessing(stdCtx, trCtx, finalResps) + n = processAndPublishEvents(trCtx, events, publisher, true, r.log) continue } - intermediateResps = append(intermediateResps, httpResp) //nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. + + // if flow of control reaches here, that means there are more than 1 request factories + // if a pagination request factory at the root level and a chain step exists, only then we will initialize flags & variables + // which are required for chaining with pagination + if r.requestFactories[i+1].isChain && r.responseProcessors[i].pagination.requestFactory != nil { + isChainExpected = true + chainIndex = i + 1 + resp, err := cloneResponse(httpResp) + if err != nil { + return err + } + initialResponse = append(initialResponse, resp) + } + intermediateResps = append(intermediateResps, httpResp) ids, err = r.getIdsFromResponses(intermediateResps, r.requestFactories[i+1].replace) if err != nil { return err } - n = r.processAndPublishEvents(stdCtx, trCtx, publisher, intermediateResps, false, i) + // we will only processAndPublishEvents here if chains do not exist, inorder to avoid unnecessary pagination + if !isChainExpected { + events := r.responseProcessors[i].startProcessing(stdCtx, trCtx, finalResps) + n = processAndPublishEvents(trCtx, events, publisher, false, r.log) + } } else { if len(ids) == 0 { n = 0 @@ -307,6 +336,14 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p } urlCopy = rf.url urlString = rf.url.String() + + // new transform context for every chain step , derived from parent transform context + var chainTrCtx *transformContext + if rf.isChain { + chainTrCtx = emptyTransformContext() + chainTrCtx.cursor = trCtx.cursor + } + // perform request over collected ids for _, id := range ids { // reformat urls of requestFactory using ids @@ -316,15 +353,15 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p } // collect data from new urls - httpResp, err = rf.collectResponse(stdCtx, trCtx, r) //nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. + httpResp, err = rf.collectResponse(stdCtx, chainTrCtx, r) if err != nil { return fmt.Errorf("failed to execute rf.collectResponse: %w", err) } // store data according to response type if i == len(r.requestFactories)-1 && len(ids) != 0 { - finalResps = append(finalResps, httpResp) //nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. + finalResps = append(finalResps, httpResp) } else { - intermediateResps = append(intermediateResps, httpResp) //nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. + intermediateResps = append(intermediateResps, httpResp) } } rf.url = urlCopy @@ -341,11 +378,22 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p } resps = intermediateResps } - n += r.processAndPublishEvents(stdCtx, trCtx, publisher, resps, i < len(r.requestFactories), i) + + var events <-chan maybeMsg + if rf.isChain { + events = rf.chainResponseProcessor.startProcessing(stdCtx, trCtx, resps) + } else { + events = r.responseProcessors[i].startProcessing(stdCtx, trCtx, resps) + } + n += processAndPublishEvents(trCtx, events, publisher, i < len(r.requestFactories), r.log) } } defer httpResp.Body.Close() + + if isChainExpected { + n += r.processRemainingChainEvents(stdCtx, trCtx, publisher, initialResponse, chainIndex) + } r.log.Infof("request finished: %d events published", n) return nil @@ -358,13 +406,17 @@ func (r *requester) getIdsFromResponses(intermediateResps []*http.Response, repl var err error // collect ids from all responses for _, resp := range intermediateResps { - defer resp.Body.Close() if resp.Body != nil { b, err = io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("error while reading response body: %w", err) } } + // gracefully close response + err = resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("error closing response body: %w", err) + } // get replace values from collected json var v interface{} @@ -396,26 +448,24 @@ func (r *requester) getIdsFromResponses(intermediateResps []*http.Response, repl return ids, nil } -// processAndPublishEvents process and publish events based on response type -func (r *requester) processAndPublishEvents(stdCtx context.Context, trCtx *transformContext, publisher inputcursor.Publisher, finalResps []*http.Response, publish bool, i int) int { - events := r.responseProcessors[i].startProcessing(stdCtx, trCtx, finalResps) - +// processAndPublishEvents process and publish events based on event type +func processAndPublishEvents(trCtx *transformContext, events <-chan maybeMsg, publisher inputcursor.Publisher, publish bool, log *logp.Logger) int { var n int for maybeMsg := range events { if maybeMsg.failed() { - r.log.Errorf("error processing response: %v", maybeMsg) + log.Errorf("error processing response: %v", maybeMsg) continue } if publish { event, err := makeEvent(maybeMsg.msg) if err != nil { - r.log.Errorf("error creating event: %v", maybeMsg) + log.Errorf("error creating event: %v", maybeMsg) continue } if err := publisher.Publish(event, trCtx.cursorMap()); err != nil { - r.log.Errorf("error publishing event: %v", err) + log.Errorf("error publishing event: %v", err) continue } } @@ -424,11 +474,157 @@ func (r *requester) processAndPublishEvents(stdCtx context.Context, trCtx *trans } trCtx.updateLastEvent(maybeMsg.msg) trCtx.updateCursor() + n++ } return n } +// processRemainingChainEvents , processes the remaining pagination events for chain blocks +func (r *requester) processRemainingChainEvents(stdCtx context.Context, trCtx *transformContext, publisher inputcursor.Publisher, initialResp []*http.Response, chainIndex int) int { + // we start from 0, and skip the 1st event since we have already processed it + events := r.responseProcessors[0].startProcessing(stdCtx, trCtx, initialResp) + + var n int + var eventCount int + for maybeMsg := range events { + if maybeMsg.failed() { + r.log.Errorf("error processing response: %v", maybeMsg) + continue + } + + if n >= 1 { // skip 1st event as it has already ben processed before + var response http.Response + response.StatusCode = 200 + body := new(bytes.Buffer) + // we construct a new response here from each of the pagination events + err := json.NewEncoder(body).Encode(maybeMsg.msg) + if err != nil { + r.log.Errorf("error processing chain event: %w", err) + continue + } + response.Body = io.NopCloser(body) + + // for each pagination response , we repeat all the chain steps / blocks + count, err := r.processChainPaginationEvents(stdCtx, trCtx, publisher, &response, chainIndex, r.log) + if err != nil { + r.log.Errorf("error processing chain event: %w", err) + continue + } + eventCount += count + + err = response.Body.Close() + if err != nil { + r.log.Errorf("error closing http response body : %w", err) + } + } + + n++ + } + return eventCount +} + +// processChainPaginationEvents takes a pagination response as input and runs all the chain blocks for the input +//nolint:bodyclose // Bad linter! The response body will always be closed by drainBody function. +func (r *requester) processChainPaginationEvents(stdCtx context.Context, trCtx *transformContext, publisher inputcursor.Publisher, response *http.Response, chainIndex int, log *logp.Logger) (int, error) { + var ( + n int + ids []string + err error + urlCopy url.URL + urlString string + httpResp *http.Response + intermediateResps []*http.Response + finalResps []*http.Response + ) + + intermediateResps = append(intermediateResps, response) + ids, err = r.getIdsFromResponses(intermediateResps, r.requestFactories[chainIndex].replace) + if err != nil { + return -1, err + } + + for i := chainIndex; i < len(r.requestFactories); i++ { + finalResps = nil + intermediateResps = nil + rf := r.requestFactories[i] + + if len(ids) == 0 { + n = 0 + continue + } + urlCopy = rf.url + urlString = rf.url.String() + + // new transform context for every chain step , derived from parent transform context + var chainTrCtx *transformContext + if rf.isChain { + chainTrCtx = emptyTransformContext() + chainTrCtx.cursor = trCtx.cursor + } + + // perform request over collected ids + for _, id := range ids { + // reformat urls of requestFactory using ids + rf.url, err = generateNewUrl(rf.replace, urlString, id) + if err != nil { + return -1, fmt.Errorf("failed to generate new URL: %w", err) + } + + // collect data from new urls + httpResp, err = rf.collectResponse(stdCtx, chainTrCtx, r) + if err != nil { + return -1, fmt.Errorf("failed to execute rf.collectResponse: %w", err) + } + // store data according to response type + if i == len(r.requestFactories)-1 && len(ids) != 0 { + finalResps = append(finalResps, httpResp) + } else { + intermediateResps = append(intermediateResps, httpResp) + } + } + rf.url = urlCopy + + var resps []*http.Response + if i == len(r.requestFactories)-1 { + resps = finalResps + } else { + // The if comdition (i < len(r.requestFactories)) ensures this branch never runs to the last element + // of r.requestFactories, therefore r.requestFactories[i+1] will never be out of bounds. + ids, err = r.getIdsFromResponses(intermediateResps, r.requestFactories[i+1].replace) + if err != nil { + return -1, err + } + resps = intermediateResps + } + events := rf.chainResponseProcessor.startProcessing(stdCtx, trCtx, resps) + n += processAndPublishEvents(trCtx, events, publisher, i < len(r.requestFactories), r.log) + } + + defer httpResp.Body.Close() + return n, nil +} + +// cloneResponse clones required http response attributes +func cloneResponse(source *http.Response) (*http.Response, error) { + var resp http.Response + + body, err := io.ReadAll(source.Body) + if err != nil { + return nil, fmt.Errorf("failed ro read http response body: %w", err) + } + + source.Body = io.NopCloser(bytes.NewReader(body)) + resp.Body = io.NopCloser(bytes.NewReader(body)) + resp.ContentLength = source.ContentLength + resp.Header = source.Header + resp.Trailer = source.Trailer + resp.StatusCode = source.StatusCode + resp.Request = source.Request.Clone(source.Request.Context()) + + return &resp, nil +} + // drainBody reads all of b to memory and then returns a equivalent // ReadCloser yielding the same bytes. // diff --git a/x-pack/filebeat/input/httpjson/response.go b/x-pack/filebeat/input/httpjson/response.go index 469b71f3ff5..f4504ce2fb6 100644 --- a/x-pack/filebeat/input/httpjson/response.go +++ b/x-pack/filebeat/input/httpjson/response.go @@ -81,10 +81,10 @@ func newResponseProcessor(config config, pagination *pagination, log *logp.Logge log: log, } // chain calls responseProcessor object - if ch.Step != nil { + if ch.Step != nil && ch.Step.Response != nil { split, _ := newSplitResponse(ch.Step.Response.Split, log) rp.split = split - } else if ch.While != nil { + } else if ch.While != nil && ch.While.Response != nil { split, _ := newSplitResponse(ch.While.Response.Split, log) rp.split = split } @@ -95,6 +95,40 @@ func newResponseProcessor(config config, pagination *pagination, log *logp.Logge return rps } +func newChainResponseProcessor(config chainConfig, httpClient *httpClient, log *logp.Logger) *responseProcessor { + pagination := &pagination{httpClient: httpClient, log: log} + + rp := &responseProcessor{ + pagination: pagination, + log: log, + } + if config.Step != nil { + if config.Step.Response == nil { + return rp + } + + ts, _ := newBasicTransformsFromConfig(config.Step.Response.Transforms, responseNamespace, log) + rp.transforms = ts + + split, _ := newSplitResponse(config.Step.Response.Split, log) + + rp.split = split + } else if config.While != nil { + if config.While.Response == nil { + return rp + } + + ts, _ := newBasicTransformsFromConfig(config.While.Response.Transforms, responseNamespace, log) + rp.transforms = ts + + split, _ := newSplitResponse(config.While.Response.Split, log) + + rp.split = split + } + + return rp +} + func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx *transformContext, resps []*http.Response) <-chan maybeMsg { trCtx.clearIntervalData() diff --git a/x-pack/filebeat/input/httpjson/transform_registry.go b/x-pack/filebeat/input/httpjson/transform_registry.go index 53caa9ded4f..26a739494db 100644 --- a/x-pack/filebeat/input/httpjson/transform_registry.go +++ b/x-pack/filebeat/input/httpjson/transform_registry.go @@ -52,7 +52,7 @@ func (reg registry) String() string { var str string for namespace, m := range reg.namespaces { - var names []string + names := make([]string, 0, len(m)) for k := range m { names = append(names, k) } diff --git a/x-pack/filebeat/input/httpjson/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go index 0b45cd4a253..5e739122b65 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -11,6 +11,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "hash" @@ -64,6 +65,7 @@ func (t *valueTpl) Unpack(in string) error { "hmac": hmacStringHex, "hmacBase64": hmacStringBase64, "join": join, + "toJSON": toJSON, "mul": mul, "now": now, "parseDate": parseDate, @@ -212,8 +214,8 @@ func parseTimestampNano(ns int64) time.Time { var regexpLinkRel = regexp.MustCompile(`<(.*)>;.*\srel\="?([^;"]*)`) -func getRFC5988Link(rel string, links []string) string { - for _, link := range links { +func getMatchLink(rel string, linksSplit []string) string { + for _, link := range linksSplit { if !regexpLinkRel.MatchString(link) { continue } @@ -229,10 +231,17 @@ func getRFC5988Link(rel string, links []string) string { return matches[1] } - return "" } +func getRFC5988Link(rel string, links []string) string { + if len(links) == 1 && strings.Count(links[0], "rel=") > 1 { + linksSplit := strings.Split(links[0], ",") + return getMatchLink(rel, linksSplit) + } + return getMatchLink(rel, links) +} + func toInt(v interface{}) int64 { vv := reflect.ValueOf(v) switch vv.Kind() { @@ -439,3 +448,12 @@ func urlEncode(value string) string { func replaceAll(old, new, s string) string { return strings.ReplaceAll(s, old, new) } + +// toJSON converts the given structure into a JSON string. +func toJSON(i interface{}) (string, error) { + result, err := json.Marshal(i) + if err != nil { + return "", fmt.Errorf("toJSON failed: %w", err) + } + return string(bytes.TrimSpace(result)), nil +} diff --git a/x-pack/filebeat/input/httpjson/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go index 300728fbf5b..5bdc92f939c 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -189,7 +189,24 @@ func TestValueTpl(t *testing.T) { expectedVal: "2020-11-05 13:25:32 +0000 UTC", }, { - name: "func getRFC5988Link", + name: "func getRFC5988Link single rel matches", + value: `[[ getRFC5988Link "next" .last_response.header.Link ]]`, + paramCtx: &transformContext{ + firstEvent: &mapstr.M{}, + lastEvent: &mapstr.M{}, + lastResponse: newTestResponse( + nil, + http.Header{"Link": []string{ + `; title="Page 3"; rel="next"`, + }}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: "https://example.com/api/v1/users?after=00ubfjQEMYBLRUWIEDKK", + }, + { + name: "func getRFC5988Link multiple rel as separate strings matches", value: `[[ getRFC5988Link "previous" .last_response.header.Link ]]`, paramCtx: &transformContext{ firstEvent: &mapstr.M{}, @@ -206,6 +223,60 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: "https://example.com/api/v1/users?before=00ubfjQEMYBLRUWIEDKK", }, + { + name: "func getRFC5988Link multiple rel as separate strings in random order matches", + value: `[[ getRFC5988Link "previous" .last_response.header.Link ]]`, + paramCtx: &transformContext{ + firstEvent: &mapstr.M{}, + lastEvent: &mapstr.M{}, + lastResponse: newTestResponse( + nil, + http.Header{"Link": []string{ + `; title="Page 1"; rel="previous"`, + `; title="Page 3"; rel="next"`, + }}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: "https://example.com/api/v1/users?before=00ubfjQEMYBLRUWIEDKK", + }, + { + name: "func getRFC5988Link multiple rel as single string matches", + value: `[[ getRFC5988Link "previous" .last_response.header.Link ]]`, + paramCtx: &transformContext{ + firstEvent: &mapstr.M{}, + lastEvent: &mapstr.M{}, + lastResponse: newTestResponse( + nil, + http.Header{"Link": []string{ + `; title="Page 1"; rel="previous", + ; title="Page 3"; rel="next"`, + }}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: "https://example.com/api/v1/users?before=00ubfjQEMYBLRUWIEDKK", + }, + { + name: "func getRFC5988Link multiple rel as single string in random order matches", + value: `[[ getRFC5988Link "next" .last_response.header.Link ]]`, + paramCtx: &transformContext{ + firstEvent: &mapstr.M{}, + lastEvent: &mapstr.M{}, + lastResponse: newTestResponse( + nil, + http.Header{"Link": []string{ + `; title="Page 1"; rel="previous", + ; title="Page 3"; rel="next"`, + }}, + "", + ), + }, + paramTr: transformable{}, + expectedVal: "https://example.com/api/v1/users?after=00ubfjQEMYBLRUWIEDKK", + }, { name: "func getRFC5988Link does not match", value: `[[ getRFC5988Link "previous" .last_response.header.Link ]]`, @@ -574,6 +645,17 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: "my value", }, + { + name: "func toJSON", + value: "[[ toJSON .first_event.events ]]", + paramCtx: &transformContext{ + firstEvent: &mapstr.M{"events": []interface{}{map[string]interface{}{"id": 1234}}}, + lastEvent: &mapstr.M{}, + lastResponse: newTestResponse(nil, nil, ""), + }, + paramTr: transformable{}, + expectedVal: `[{"id":1234}]`, + }, } for _, tc := range cases { diff --git a/x-pack/filebeat/input/lumberjack/_meta/fields.yml b/x-pack/filebeat/input/lumberjack/_meta/fields.yml new file mode 100644 index 00000000000..ee3ef012006 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/_meta/fields.yml @@ -0,0 +1,9 @@ +- key: lumberjack + title: "Lumberjack" + description: > + Fields from Lumberjack input. + fields: + - name: lumberjack + type: flattened + description: > + Structured data received in an event sent over the Lumberjack protocol. diff --git a/x-pack/filebeat/input/lumberjack/ack.go b/x-pack/filebeat/input/lumberjack/ack.go new file mode 100644 index 00000000000..ab15ad157dc --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/ack.go @@ -0,0 +1,78 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "sync" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/acker" +) + +// batchACKTracker invokes batchACK when all events associated to the batch +// have been published and acknowledged by an output. +type batchACKTracker struct { + batchACK func() + + mutex sync.Mutex // mutex synchronizes access to pendingACKs. + pendingACKs int64 // Number of Beat events in lumberjack batch that are pending ACKs. +} + +// newBatchACKTracker returns a new batchACKTracker. The provided batchACK function +// is invoked after the full batch has been acknowledged. Ready() must be invoked +// after all events in the batch are published. +func newBatchACKTracker(batchACKCallback func()) *batchACKTracker { + return &batchACKTracker{ + batchACK: batchACKCallback, + pendingACKs: 1, // Ready() must be called to consume this "1". + } +} + +// Ready signals that the batch has been fully consumed. Only +// after the batch is marked as "ready" can the lumberjack batch +// be ACKed. This prevents the batch from being ACKed prematurely. +func (t *batchACKTracker) Ready() { + t.ACK() +} + +// Add increments the number of pending ACKs. +func (t *batchACKTracker) Add() { + t.mutex.Lock() + defer t.mutex.Unlock() + + t.pendingACKs++ +} + +// ACK decrements the number of pending event ACKs. When all pending ACKs are +// received then the lumberjack batch is ACKed. +func (t *batchACKTracker) ACK() { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.pendingACKs <= 0 { + panic("misuse detected: negative ACK counter") + } + + t.pendingACKs-- + if t.pendingACKs == 0 { + t.batchACK() + } +} + +// newEventACKHandler returns a beat ACKer that can receive callbacks when +// an event has been ACKed an output. If the event contains a private metadata +// pointing to a batchACKTracker then it will invoke the tracker's ACK() method +// to decrement the number of pending ACKs. +func newEventACKHandler() beat.ACKer { + return acker.ConnectionOnly( + acker.EventPrivateReporter(func(_ int, privates []interface{}) { + for _, private := range privates { + if ack, ok := private.(*batchACKTracker); ok { + ack.ACK() + } + } + }), + ) +} diff --git a/x-pack/filebeat/input/lumberjack/ack_test.go b/x-pack/filebeat/input/lumberjack/ack_test.go new file mode 100644 index 00000000000..90e03819488 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/ack_test.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/go-lumber/lj" +) + +func TestBatchACKTracker(t *testing.T) { + t.Run("empty", func(t *testing.T) { + batch := lj.NewBatch(nil) + + acker := newBatchACKTracker(batch.ACK) + require.False(t, isACKed(batch)) + + acker.Ready() + require.True(t, isACKed(batch)) + }) + + t.Run("single_event", func(t *testing.T) { + batch := lj.NewBatch(nil) + + acker := newBatchACKTracker(batch.ACK) + acker.Add() + acker.ACK() + require.False(t, isACKed(batch)) + + acker.Ready() + require.True(t, isACKed(batch)) + }) +} + +func isACKed(batch *lj.Batch) bool { + select { + case <-batch.Await(): + return true + default: + return false + } +} diff --git a/x-pack/filebeat/input/lumberjack/config.go b/x-pack/filebeat/input/lumberjack/config.go new file mode 100644 index 00000000000..53ceed2f8ce --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/config.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "fmt" + "strings" + "time" + + "github.com/elastic/elastic-agent-libs/transport/tlscommon" +) + +type config struct { + ListenAddress string `config:"listen_address" validate:"nonzero"` // Bind address for the server (e.g. address:port). Default to localhost:5044. + Versions []string `config:"versions"` // List of Lumberjack version (e.g. v1, v2). + TLS *tlscommon.ServerConfig `config:"ssl"` // TLS options. + Keepalive time.Duration `config:"keepalive" validate:"min=0"` // Keepalive interval for notifying clients that batches that are not yet ACKed. + Timeout time.Duration `config:"timeout" validate:"min=0"` // Read / write timeouts for Lumberjack server. + MaxConnections int `config:"max_connections" validate:"min=0"` // Maximum number of concurrent connections. Default is 0 which means no limit. +} + +func (c *config) InitDefaults() { + c.ListenAddress = "localhost:5044" + c.Versions = []string{"v1", "v2"} +} + +func (c *config) Validate() error { + for _, v := range c.Versions { + switch strings.ToLower(v) { + case "v1", "v2": + default: + return fmt.Errorf("invalid lumberjack version %q: allowed values are v1 and v2", v) + } + } + + return nil +} diff --git a/x-pack/filebeat/input/lumberjack/config_test.go b/x-pack/filebeat/input/lumberjack/config_test.go new file mode 100644 index 00000000000..5b9e73d4d7c --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/config_test.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "testing" + + "github.com/stretchr/testify/require" + + conf "github.com/elastic/elastic-agent-libs/config" +) + +func TestConfig(t *testing.T) { + testCases := []struct { + name string + userConfig map[string]interface{} + expected *config + expectedErr string + }{ + { + "defaults", + map[string]interface{}{}, + &config{ + ListenAddress: "localhost:5044", + Versions: []string{"v1", "v2"}, + }, + "", + }, + { + "validate version", + map[string]interface{}{ + "versions": []string{"v3"}, + }, + nil, + `invalid lumberjack version "v3"`, + }, + { + "validate keepalive", + map[string]interface{}{ + "keepalive": "-1s", + }, + nil, + `requires duration >= 0`, + }, + { + "validate max_connections", + map[string]interface{}{ + "max_connections": -1, + }, + nil, + `requires value >= 0 accessing 'max_connections'`, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + c := conf.MustNewConfigFrom(tc.userConfig) + + var ljConf config + err := c.Unpack(&ljConf) + + if tc.expectedErr != "" { + require.Error(t, err, "expected error: %s", tc.expectedErr) + require.Contains(t, err.Error(), tc.expectedErr) + return + } + + require.Equal(t, *tc.expected, ljConf) + }) + } +} diff --git a/x-pack/filebeat/input/lumberjack/fields.go b/x-pack/filebeat/input/lumberjack/fields.go new file mode 100644 index 00000000000..d54be5d16eb --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package lumberjack + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "lumberjack", asset.ModuleFieldsPri, AssetLumberjack); err != nil { + panic(err) + } +} + +// AssetLumberjack returns asset data. +// This is the base64 encoded zlib format compressed contents of input/lumberjack. +func AssetLumberjack() string { + return "eJxsjjEOwjAQBHu/YpU+eYALSio6XmDsjTBxbOtyjpTfo0QIEGKLK/ZGmu0xcbNIbb5RHs5PBtCoiRbd5V12BghcvMSqsWSLkwGAc2QKC0YpMz4wYq5NBwOMx98ebI/sZv6Y9uhWaTEmp8rM8Gr/2PZcVZrXJgwITh2EnnFlQMxwGVyZFct+ykqB3vk9rErR4ksazDMAAP//JmxQDQ==" +} diff --git a/x-pack/filebeat/input/lumberjack/generate_certs_test.go b/x-pack/filebeat/input/lumberjack/generate_certs_test.go new file mode 100644 index 00000000000..e66eb1b5b8b --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/generate_certs_test.go @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "testing" + "time" +) + +type Cert struct { + signedCertDER []byte // DER encoded certificate from x509.CreateCertificate. + key *rsa.PrivateKey // RSA public / private key pair. +} + +// CertPEM returns the cert encoded as PEM. +func (c Cert) CertPEM(t testing.TB) []byte { return pemEncode(t, c.signedCertDER, "CERTIFICATE") } + +// KeyPEM returns the private key encoded as PEM. +func (c Cert) KeyPEM(t testing.TB) []byte { + return pemEncode(t, x509.MarshalPKCS1PrivateKey(c.key), "RSA PRIVATE KEY") +} + +func (c Cert) TLSCertificate(t testing.TB) tls.Certificate { + pair, err := tls.X509KeyPair(c.CertPEM(t), c.KeyPEM(t)) + if err != nil { + t.Fatal(err) + } + + return pair +} + +// generateCertData creates a root CA, server, and client cert suitable for +// testing mTLS. +func generateCertData(t testing.TB) (rootCA, client, server Cert) { + t.Helper() + + // CA cert + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + Organization: []string{"Elastic"}, + Country: []string{"US"}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"West El Camino Real"}, + PostalCode: []string{"94040"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + var err error + rootCA.key, err = rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Fatal(err) + } + + rootCA.signedCertDER, err = x509.CreateCertificate(rand.Reader, ca, ca, &rootCA.key.PublicKey, rootCA.key) + if err != nil { + t.Fatal(err) + } + + // Server cert + { + // set up our server certificate + serverCert := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{ + Organization: []string{"Elastic"}, + Country: []string{"US"}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"West El Camino Real"}, + PostalCode: []string{"94040"}, + CommonName: "server", + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + DNSNames: []string{"localhost"}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + SubjectKeyId: []byte{1, 2, 3, 4, 5}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + server.key, err = rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Fatal(err) + } + + server.signedCertDER, err = x509.CreateCertificate(rand.Reader, serverCert, ca, &server.key.PublicKey, rootCA.key) + if err != nil { + t.Fatal(err) + } + } + + // Client cert. + { + clientCert := &x509.Certificate{ + SerialNumber: big.NewInt(3), + Subject: pkix.Name{ + Organization: []string{"Elastic"}, + Country: []string{"US"}, + Locality: []string{"San Francisco"}, + StreetAddress: []string{"West El Camino Real"}, + PostalCode: []string{"94040"}, + CommonName: "client", + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + SubjectKeyId: []byte{1, 2, 3, 4, 5}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + EmailAddresses: []string{"client@example.com"}, + } + + client.key, err = rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + t.Fatal(err) + } + + client.signedCertDER, err = x509.CreateCertificate(rand.Reader, clientCert, ca, &client.key.PublicKey, rootCA.key) + if err != nil { + t.Fatal(err) + } + } + + return rootCA, client, server +} + +func pemEncode(t testing.TB, certBytes []byte, certType string) []byte { + t.Helper() + + pemData := new(bytes.Buffer) + if err := pem.Encode(pemData, &pem.Block{Type: certType, Bytes: certBytes}); err != nil { + t.Fatal(err) + } + + return pemData.Bytes() +} diff --git a/x-pack/filebeat/input/lumberjack/input.go b/x-pack/filebeat/input/lumberjack/input.go new file mode 100644 index 00000000000..9471bb35e92 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/input.go @@ -0,0 +1,94 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "fmt" + + inputv2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/feature" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/monitoring" +) + +const ( + inputName = "lumberjack" +) + +func Plugin() inputv2.Plugin { + return inputv2.Plugin{ + Name: inputName, + Stability: feature.Beta, + Info: "Receives data streamed via the Lumberjack protocol.", + Manager: inputv2.ConfigureWith(configure), + } +} + +func configure(cfg *conf.C) (inputv2.Input, error) { + var lumberjackConfig config + if err := cfg.Unpack(&lumberjackConfig); err != nil { + return nil, err + } + + return newLumberjackInput(lumberjackConfig) +} + +// lumberjackInput implements the Filebeat input V2 interface. The input is stateless. +type lumberjackInput struct { + config config +} + +var _ inputv2.Input = (*lumberjackInput)(nil) + +func newLumberjackInput(lumberjackConfig config) (*lumberjackInput, error) { + return &lumberjackInput{config: lumberjackConfig}, nil +} + +func (i *lumberjackInput) Name() string { return inputName } + +func (i *lumberjackInput) Test(inputCtx inputv2.TestContext) error { + s, err := newServer(i.config, inputCtx.Logger, nil, nil) + if err != nil { + return err + } + return s.Close() +} + +func (i *lumberjackInput) Run(inputCtx inputv2.Context, pipeline beat.Pipeline) error { + inputCtx.Logger.Info("Starting " + inputName + " input") + defer inputCtx.Logger.Info(inputName + " input stopped") + + // Create client for publishing events and receive notification of their ACKs. + client, err := pipeline.ConnectWith(beat.ClientConfig{ + CloseRef: inputCtx.Cancelation, + ACKHandler: newEventACKHandler(), + }) + if err != nil { + return fmt.Errorf("failed to create pipeline client: %w", err) + } + defer client.Close() + + setGoLumberLogger(inputCtx.Logger.Named("go-lumber")) + + metricRegistry := monitoring.GetNamespace("dataset").GetRegistry() + metrics := newInputMetrics(metricRegistry, inputCtx.ID) + defer metrics.Close() + + s, err := newServer(i.config, inputCtx.Logger, client.Publish, metrics) + if err != nil { + return err + } + defer s.Close() + + // Shutdown the server when cancellation is signaled. + go func() { + <-inputCtx.Cancelation.Done() + s.Close() + }() + + // Run server until the cancellation signal. + return s.Run() +} diff --git a/x-pack/filebeat/input/lumberjack/logger.go b/x-pack/filebeat/input/lumberjack/logger.go new file mode 100644 index 00000000000..0f15b2b0d11 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/logger.go @@ -0,0 +1,40 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "sync" + + "go.uber.org/zap" + + "github.com/elastic/elastic-agent-libs/logp" + lumberlog "github.com/elastic/go-lumber/log" +) + +var setGoLumberLoggerOnce sync.Once + +func setGoLumberLogger(parent *logp.Logger) { + setGoLumberLoggerOnce.Do(func() { + lumberlog.Logger = &goLumberLogger{parent: parent.WithOptions(zap.AddCallerSkip(2))} + }) +} + +// goLumberLogger implements the go-lumber/log.Logging interface to route +// log message from go-lumber to Beats logp. +type goLumberLogger struct { + parent *logp.Logger +} + +func (l *goLumberLogger) Printf(s string, i ...interface{}) { + l.parent.Debugf(s, i...) +} + +func (l *goLumberLogger) Println(i ...interface{}) { + l.parent.Debug(i...) +} + +func (l *goLumberLogger) Print(i ...interface{}) { + l.parent.Debug(i...) +} diff --git a/x-pack/filebeat/input/lumberjack/metrics.go b/x-pack/filebeat/input/lumberjack/metrics.go new file mode 100644 index 00000000000..ebceeb397b7 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/metrics.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "github.com/rcrowley/go-metrics" + + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/monitoring/adapter" +) + +type inputMetrics struct { + id string // Input ID. + parent *monitoring.Registry // Parent registry holding this input's ID as a key. + bindAddress *monitoring.String // Bind address of input. + + batchesReceivedTotal *monitoring.Uint // Number of Lumberjack batches received (not necessarily processed fully). + batchesACKedTotal *monitoring.Uint // Number of Lumberjack batches ACKed. + messagesReceivedTotal *monitoring.Uint // Number of Lumberjack messages received (not necessarily processed fully). + batchProcessingTime metrics.Sample // Histogram of the elapsed batch processing times in nanoseconds (time of receipt to time of ACK for non-empty batches). +} + +// Close removes the metrics from the registry. +func (m *inputMetrics) Close() { + m.parent.Remove(m.id) +} + +func newInputMetrics(parent *monitoring.Registry, id string) *inputMetrics { + reg := parent.NewRegistry(id) + monitoring.NewString(reg, "input").Set(inputName) + monitoring.NewString(reg, "id").Set(id) + out := &inputMetrics{ + id: id, + parent: reg, + bindAddress: monitoring.NewString(reg, "bind_address"), + batchesReceivedTotal: monitoring.NewUint(reg, "batches_received_total"), + batchesACKedTotal: monitoring.NewUint(reg, "batches_acked_total"), + messagesReceivedTotal: monitoring.NewUint(reg, "messages_received_total"), + batchProcessingTime: metrics.NewUniformSample(1024), + } + adapter.NewGoMetrics(reg, "batch_processing_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.batchProcessingTime)) //nolint:errcheck // A unique namespace is used so name collisions are impossible. + + return out +} diff --git a/x-pack/filebeat/input/lumberjack/server.go b/x-pack/filebeat/input/lumberjack/server.go new file mode 100644 index 00000000000..96d0366e2b5 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/server.go @@ -0,0 +1,182 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "crypto/tls" + "net" + "strings" + "sync" + "time" + + "golang.org/x/net/netutil" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" + lumber "github.com/elastic/go-lumber/server" +) + +type server struct { + config config + log *logp.Logger + publish func(beat.Event) + metrics *inputMetrics + ljSvr lumber.Server + ljSvrCloseOnce sync.Once + bindAddress string +} + +func newServer(c config, log *logp.Logger, pub func(beat.Event), metrics *inputMetrics) (*server, error) { + ljSvr, bindAddress, err := newLumberjack(c) + if err != nil { + return nil, err + } + + if metrics == nil { + metrics = newInputMetrics(monitoring.NewRegistry(), "") + } + + bindURI := "tcp://" + bindAddress + if c.TLS.IsEnabled() { + bindURI = "tls://" + bindAddress + } + log.Infof(inputName+" is listening at %v.", bindURI) + metrics.bindAddress.Set(bindURI) + + return &server{ + config: c, + log: log, + publish: pub, + metrics: metrics, + ljSvr: ljSvr, + bindAddress: bindAddress, + }, nil +} + +func (s *server) Close() error { + var err error + s.ljSvrCloseOnce.Do(func() { + err = s.ljSvr.Close() + }) + return err +} + +func (s *server) Run() error { + // Process batches until the input is stopped. + for batch := range s.ljSvr.ReceiveChan() { + s.metrics.batchesReceivedTotal.Inc() + + if len(batch.Events) == 0 { + batch.ACK() + s.metrics.batchesACKedTotal.Inc() + continue + } + s.metrics.messagesReceivedTotal.Add(uint64(len(batch.Events))) + + // Track all the Beat events associated to the Lumberjack batch so that + // the batch can be ACKed after the Beat events are delivered successfully. + start := time.Now() + acker := newBatchACKTracker(func() { + batch.ACK() + s.metrics.batchesACKedTotal.Inc() + s.metrics.batchProcessingTime.Update(time.Since(start).Nanoseconds()) + }) + + for _, ljEvent := range batch.Events { + acker.Add() + s.publish(makeEvent(batch.RemoteAddr, batch.TLS, ljEvent, acker)) + } + + // Mark the batch as "ready" after Beat events are generated for each + // Lumberjack event. + acker.Ready() + } + + return nil +} + +func makeEvent(remoteAddr string, tlsState *tls.ConnectionState, lumberjackEvent interface{}, acker *batchACKTracker) beat.Event { + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: map[string]interface{}{ + "source": map[string]interface{}{ + "address": remoteAddr, + }, + "lumberjack": lumberjackEvent, + }, + Private: acker, + } + + if tlsState != nil && len(tlsState.PeerCertificates) > 0 { + event.Fields["tls"] = map[string]interface{}{ + "client": map[string]interface{}{ + "subject": tlsState.PeerCertificates[0].Subject.CommonName, + }, + } + } + + return event +} + +func newLumberjack(c config) (lj lumber.Server, bindAddress string, err error) { + // Setup optional TLS. + var tlsConfig *tls.Config + if c.TLS.IsEnabled() { + elasticTLSConfig, err := tlscommon.LoadTLSServerConfig(c.TLS) + if err != nil { + return nil, "", err + } + + // NOTE: Passing an empty string disables checking the client certificate for a + // specific hostname. + tlsConfig = elasticTLSConfig.BuildServerConfig("") + } + + // Start listener. + l, err := net.Listen("tcp", c.ListenAddress) + if err != nil { + return nil, "", err + } + if tlsConfig != nil { + l = tls.NewListener(l, tlsConfig) + } + if c.MaxConnections > 0 { + l = netutil.LimitListener(l, c.MaxConnections) + } + + // Start lumberjack server. + s, err := lumber.NewWithListener(l, makeLumberjackOptions(c)...) + if err != nil { + return nil, "", err + } + + return s, l.Addr().String(), nil +} + +func makeLumberjackOptions(c config) []lumber.Option { + var opts []lumber.Option + + // Versions + for _, p := range c.Versions { + switch strings.ToLower(p) { + case "v1": + opts = append(opts, lumber.V1(true)) + case "v2": + opts = append(opts, lumber.V2(true)) + } + } + + if c.Keepalive > 0 { + opts = append(opts, lumber.Keepalive(c.Keepalive)) + } + + if c.Timeout > 0 { + opts = append(opts, lumber.Timeout(c.Keepalive)) + } + + return opts +} diff --git a/x-pack/filebeat/input/lumberjack/server_test.go b/x-pack/filebeat/input/lumberjack/server_test.go new file mode 100644 index 00000000000..971a37f7255 --- /dev/null +++ b/x-pack/filebeat/input/lumberjack/server_test.go @@ -0,0 +1,238 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package lumberjack + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" + client "github.com/elastic/go-lumber/client/v2" +) + +const testTimeout = 10 * time.Second + +func TestServer(t *testing.T) { + makeTestConfig := func() config { + var c config + c.InitDefaults() + c.ListenAddress = "localhost:0" + c.MaxConnections = 1 + c.Keepalive = time.Second + c.Timeout = time.Second + return c + } + + t.Run("empty_batch", func(t *testing.T) { + testSendReceive(t, makeTestConfig(), 0, nil) + }) + + t.Run("no tls", func(t *testing.T) { + testSendReceive(t, makeTestConfig(), 10, nil) + }) + + t.Run("tls", func(t *testing.T) { + clientConf, serverConf := tlsSetup(t) + clientConf.Certificates = nil + + c := makeTestConfig() + c.TLS = serverConf + // Disable mTLS requirements in the server. + c.TLS.ClientAuth = 0 // tls.NoClientCert + c.TLS.VerificationMode = tlscommon.VerifyNone + + testSendReceive(t, c, 10, clientConf) + }) + + t.Run("mutual tls", func(t *testing.T) { + clientConf, serverConf := tlsSetup(t) + + c := makeTestConfig() + c.TLS = serverConf + + testSendReceive(t, c, 10, clientConf) + }) +} + +func testSendReceive(t testing.TB, c config, numberOfEvents int, clientTLSConfig *tls.Config) { + require.NoError(t, logp.TestingSetup()) + log := logp.NewLogger(inputName).With("test_name", t.Name()) + + ctx, shutdown := context.WithTimeout(context.Background(), testTimeout) + t.Cleanup(shutdown) + collect := newEventCollector(ctx, numberOfEvents) + + // Start server. + s, err := newServer(c, log, collect.Publish, nil) + require.NoError(t, err) + go func() { + <-ctx.Done() + s.Close() + }() + + // Asynchronously send and receive events. + var wg errgroup.Group + wg.Go(s.Run) + wg.Go(func() error { + // The client returns on error or after an E2E ACK is received. + // In both cases the test should shutdown. + defer shutdown() + + return sendData(ctx, t, s.bindAddress, numberOfEvents, clientTLSConfig) + }) + + // Wait for the expected number of events. + collect.Await(t) + + // Check for errors from client and server. + require.NoError(t, wg.Wait()) +} + +func sendData(ctx context.Context, t testing.TB, bindAddress string, numberOfEvents int, clientTLSConfig *tls.Config) error { + _, port, err := net.SplitHostPort(bindAddress) + if err != nil { + return err + } + + dialFunc := net.Dial + if clientTLSConfig != nil { + dialer := &tls.Dialer{ + Config: clientTLSConfig, + } + dialFunc = dialer.Dial + } + + c, err := client.SyncDialWith(dialFunc, net.JoinHostPort("localhost", port)) + if err != nil { + return fmt.Errorf("client dial error: %w", err) + } + defer c.Close() + go func() { + <-ctx.Done() + c.Close() + }() + t.Log("Lumberjack client connected.") + + var events []interface{} + for i := 0; i < numberOfEvents; i++ { + events = append(events, map[string]interface{}{ + "message": "hello world!", + "index": i, + }) + } + + if _, err = c.Send(events); err != nil { + return fmt.Errorf("failed sending lumberjack events: %w", err) + } + t.Log("Lumberjack client sent", len(events), "events.") + + return nil +} + +type eventCollector struct { + sync.Mutex + events []beat.Event + awaitCtx context.Context // awaitCtx is cancelled when events length is expectedSize. + awaitCancel context.CancelFunc + expectedSize int +} + +func newEventCollector(ctx context.Context, expectedSize int) *eventCollector { + ctx, cancel := context.WithCancel(ctx) + if expectedSize == 0 { + cancel() + } + + return &eventCollector{ + awaitCtx: ctx, + awaitCancel: cancel, + expectedSize: expectedSize, + } +} + +func (c *eventCollector) Publish(evt beat.Event) { + c.Lock() + defer c.Unlock() + + c.events = append(c.events, evt) + evt.Private.(*batchACKTracker).ACK() + + if len(c.events) == c.expectedSize { + c.awaitCancel() + } +} + +func (c *eventCollector) Await(t testing.TB) []beat.Event { + t.Helper() + + <-c.awaitCtx.Done() + if errors.Is(c.awaitCtx.Err(), context.DeadlineExceeded) { + t.Fatal(c.awaitCtx.Err()) + } + + c.Lock() + defer c.Unlock() + + if len(c.events) > c.expectedSize { + t.Fatalf("more events received than expected, got %d, want %d", len(c.events), c.expectedSize) + } + + events := make([]beat.Event, len(c.events)) + copy(events, c.events) + return events +} + +var ( + certDataOnce sync.Once + certData = struct { + ca, client, server Cert + }{} +) + +// tlsSetup return client and server configurations ready to test mutual TLS. +func tlsSetup(t *testing.T) (clientConfig *tls.Config, serverConfig *tlscommon.ServerConfig) { + t.Helper() + + certDataOnce.Do(func() { + certData.ca, certData.client, certData.server = generateCertData(t) + }) + + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(certData.ca.CertPEM(t)) + + clientConfig = &tls.Config{ + RootCAs: certPool, + Certificates: []tls.Certificate{certData.client.TLSCertificate(t)}, + MinVersion: tls.VersionTLS12, + } + + serverConfig = &tlscommon.ServerConfig{ + // NOTE: VerifyCertificate is ineffective unless ClientAuth is set to RequireAndVerifyClientCert. + VerificationMode: tlscommon.VerifyCertificate, + // Unfortunately ServerConfig uses an unexported type in an exported field. + ClientAuth: 4, // tls.RequireAndVerifyClientCert + CAs: []string{ + string(certData.ca.CertPEM(t)), + }, + Certificate: tlscommon.CertificateConfig{ + Certificate: string(certData.server.CertPEM(t)), + Key: string(certData.server.KeyPEM(t)), + }, + } + + return clientConfig, serverConfig +} diff --git a/x-pack/filebeat/input/netflow/convert.go b/x-pack/filebeat/input/netflow/convert.go index 7fcbdc84419..87bbf94788b 100644 --- a/x-pack/filebeat/input/netflow/convert.go +++ b/x-pack/filebeat/input/netflow/convert.go @@ -391,10 +391,27 @@ func getKeyIP(dict record.Map, key string) (value net.IP, found bool) { // because HardwareAddr doesn't implement Marshaler interface. func fixMacAddresses(dict map[string]interface{}) { for key, value := range dict { - if asMac, ok := value.(net.HardwareAddr); ok { - dict[key] = asMac.String() + if addr, ok := value.(net.HardwareAddr); ok { + if len(addr) == 0 { + delete(dict, key) + continue + } + dict[key] = formatHardwareAddr(addr) + } + } +} + +// formatHardwareAddr formats hardware addresses according to the ECS spec. +func formatHardwareAddr(addr net.HardwareAddr) string { + buf := make([]byte, 0, len(addr)*3-1) + for _, b := range addr { + if len(buf) != 0 { + buf = append(buf, '-') } + const hexDigit = "0123456789ABCDEF" + buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf]) } + return string(buf) } // Locality is an enum representing the locality of a network address. diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json index b1dd7eb6c1d..3fa4549af7d 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json @@ -57,7 +57,7 @@ "reason_text": "Normal Operation", "service_name": "https", "source_ipv4_address": "10.236.5.4", - "source_mac_address": "00:50:56:b9:26:46", + "source_mac_address": "00-50-56-B9-26-46", "source_transport_port": 51917, "timestamp": 1524039407, "traffic_type": 0, @@ -84,7 +84,7 @@ "bytes": 0, "ip": "10.236.5.4", "locality": "internal", - "mac": "00:50:56:b9:26:46", + "mac": "00-50-56-B9-26-46", "packets": 0, "port": 51917 } @@ -148,7 +148,7 @@ "reason_text": "Normal Operation", "service_name": "https", "source_ipv4_address": "64.235.151.76", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 443, "timestamp": 1524039407, "traffic_type": 0, @@ -175,7 +175,7 @@ "bytes": 0, "ip": "64.235.151.76", "locality": "external", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 0, "port": 443 } diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json index 7d89a8dad1e..4eb6dfb9a27 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json @@ -47,7 +47,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.99.130.239", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 65105, "type": "netflow_flow" }, @@ -72,7 +72,7 @@ "bytes": 0, "ip": "10.99.130.239", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 0, "port": 65105 } @@ -126,7 +126,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.99.252.50", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 53, "type": "netflow_flow" }, @@ -151,7 +151,7 @@ "bytes": 81, "ip": "10.99.252.50", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 1, "port": 53 } @@ -205,7 +205,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.99.130.239", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 65105, "type": "netflow_flow" }, @@ -230,7 +230,7 @@ "bytes": 0, "ip": "10.99.130.239", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 0, "port": 65105 } @@ -284,7 +284,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.98.243.20", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 53, "type": "netflow_flow" }, @@ -309,7 +309,7 @@ "bytes": 81, "ip": "10.98.243.20", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 1, "port": 53 } @@ -363,7 +363,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.99.168.140", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 52344, "type": "netflow_flow" }, @@ -388,7 +388,7 @@ "bytes": 0, "ip": "10.99.168.140", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 0, "port": 52344 } @@ -442,7 +442,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.98.243.20", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 53, "type": "netflow_flow" }, @@ -467,7 +467,7 @@ "bytes": 113, "ip": "10.98.243.20", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 1, "port": 53 } @@ -521,7 +521,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.99.168.140", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 50294, "type": "netflow_flow" }, @@ -546,7 +546,7 @@ "bytes": 0, "ip": "10.99.168.140", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 0, "port": 50294 } @@ -600,7 +600,7 @@ "packet_total_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.98.243.20", - "source_mac_address": "00:00:00:00:00:00", + "source_mac_address": "00-00-00-00-00-00", "source_transport_port": 53, "type": "netflow_flow" }, @@ -625,7 +625,7 @@ "bytes": 113, "ip": "10.98.243.20", "locality": "internal", - "mac": "00:00:00:00:00:00", + "mac": "00-00-00-00-00-00", "packets": 1, "port": 53 } diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json index d8520437747..cf60df28120 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json @@ -44,7 +44,7 @@ "packet_delta_count": 1, "protocol_identifier": 17, "source_ipv4_address": "192.168.0.111", - "source_mac_address": "ec:1f:72:11:9f:c1", + "source_mac_address": "EC-1F-72-11-9F-C1", "source_transport_port": 37301, "tcp_control_bits": 0, "type": "netflow_flow" @@ -70,7 +70,7 @@ "bytes": 75, "ip": "192.168.0.111", "locality": "internal", - "mac": "ec:1f:72:11:9f:c1", + "mac": "EC-1F-72-11-9F-C1", "packets": 1, "port": 37301 } @@ -121,7 +121,7 @@ "packet_delta_count": 1, "protocol_identifier": 17, "source_ipv4_address": "192.168.0.111", - "source_mac_address": "ec:1f:72:11:9f:c1", + "source_mac_address": "EC-1F-72-11-9F-C1", "source_transport_port": 58411, "tcp_control_bits": 0, "type": "netflow_flow" @@ -147,7 +147,7 @@ "bytes": 75, "ip": "192.168.0.111", "locality": "internal", - "mac": "ec:1f:72:11:9f:c1", + "mac": "EC-1F-72-11-9F-C1", "packets": 1, "port": 58411 } @@ -198,7 +198,7 @@ "packet_delta_count": 1, "protocol_identifier": 17, "source_ipv4_address": "192.168.0.111", - "source_mac_address": "ec:1f:72:11:9f:c1", + "source_mac_address": "EC-1F-72-11-9F-C1", "source_transport_port": 37661, "tcp_control_bits": 0, "type": "netflow_flow" @@ -224,7 +224,7 @@ "bytes": 75, "ip": "192.168.0.111", "locality": "internal", - "mac": "ec:1f:72:11:9f:c1", + "mac": "EC-1F-72-11-9F-C1", "packets": 1, "port": 37661 } @@ -275,7 +275,7 @@ "packet_delta_count": 1, "protocol_identifier": 17, "source_ipv4_address": "192.168.0.111", - "source_mac_address": "ec:1f:72:11:9f:c1", + "source_mac_address": "EC-1F-72-11-9F-C1", "source_transport_port": 60212, "tcp_control_bits": 0, "type": "netflow_flow" @@ -301,7 +301,7 @@ "bytes": 75, "ip": "192.168.0.111", "locality": "internal", - "mac": "ec:1f:72:11:9f:c1", + "mac": "EC-1F-72-11-9F-C1", "packets": 1, "port": 60212 } @@ -352,7 +352,7 @@ "packet_delta_count": 10, "protocol_identifier": 6, "source_ipv4_address": "158.85.58.115", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 5222, "tcp_control_bits": 29, "type": "netflow_flow" @@ -378,7 +378,7 @@ "bytes": 964, "ip": "158.85.58.115", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 10, "port": 5222 } @@ -429,7 +429,7 @@ "packet_delta_count": 8, "protocol_identifier": 17, "source_ipv4_address": "192.168.0.88", - "source_mac_address": "a4:d1:8c:e9:30:2c", + "source_mac_address": "A4-D1-8C-E9-30-2C", "source_transport_port": 61490, "tcp_control_bits": 0, "type": "netflow_flow" @@ -455,7 +455,7 @@ "bytes": 2748, "ip": "192.168.0.88", "locality": "internal", - "mac": "a4:d1:8c:e9:30:2c", + "mac": "A4-D1-8C-E9-30-2C", "packets": 8, "port": 61490 } @@ -506,7 +506,7 @@ "packet_delta_count": 9, "protocol_identifier": 17, "source_ipv4_address": "216.58.212.195", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 0, "type": "netflow_flow" @@ -532,7 +532,7 @@ "bytes": 2023, "ip": "216.58.212.195", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 9, "port": 443 } @@ -583,7 +583,7 @@ "packet_delta_count": 9, "protocol_identifier": 6, "source_ipv4_address": "192.168.1.201", - "source_mac_address": "98:01:a7:9f:8d:5f", + "source_mac_address": "98-01-A7-9F-8D-5F", "source_transport_port": 50299, "tcp_control_bits": 24, "type": "netflow_flow" @@ -609,7 +609,7 @@ "bytes": 2180, "ip": "192.168.1.201", "locality": "internal", - "mac": "98:01:a7:9f:8d:5f", + "mac": "98-01-A7-9F-8D-5F", "packets": 9, "port": 50299 } @@ -660,7 +660,7 @@ "packet_delta_count": 9, "protocol_identifier": 6, "source_ipv4_address": "216.58.201.106", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 24, "type": "netflow_flow" @@ -686,7 +686,7 @@ "bytes": 700, "ip": "216.58.201.106", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 9, "port": 443 } @@ -737,7 +737,7 @@ "packet_delta_count": 2, "protocol_identifier": 6, "source_ipv4_address": "52.236.33.163", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 24, "type": "netflow_flow" @@ -763,7 +763,7 @@ "bytes": 161, "ip": "52.236.33.163", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 2, "port": 443 } @@ -814,7 +814,7 @@ "packet_delta_count": 21, "protocol_identifier": 6, "source_ipv4_address": "192.168.3.34", - "source_mac_address": "1c:5c:f2:07:0f:2a", + "source_mac_address": "1C-5C-F2-07-0F-2A", "source_transport_port": 61674, "tcp_control_bits": 27, "type": "netflow_flow" @@ -840,7 +840,7 @@ "bytes": 1764, "ip": "192.168.3.34", "locality": "internal", - "mac": "1c:5c:f2:07:0f:2a", + "mac": "1C-5C-F2-07-0F-2A", "packets": 21, "port": 61674 } @@ -891,7 +891,7 @@ "packet_delta_count": 30, "protocol_identifier": 6, "source_ipv4_address": "209.197.3.19", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 31, "type": "netflow_flow" @@ -917,7 +917,7 @@ "bytes": 13811, "ip": "209.197.3.19", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 30, "port": 443 } @@ -968,7 +968,7 @@ "packet_delta_count": 16, "protocol_identifier": 6, "source_ipv4_address": "52.216.130.237", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 27, "type": "netflow_flow" @@ -994,7 +994,7 @@ "bytes": 4717, "ip": "52.216.130.237", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 16, "port": 443 } @@ -1045,7 +1045,7 @@ "packet_delta_count": 13, "protocol_identifier": 6, "source_ipv4_address": "192.168.0.157", - "source_mac_address": "b0:34:95:0d:d2:5d", + "source_mac_address": "B0-34-95-0D-D2-5D", "source_transport_port": 51209, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1071,7 +1071,7 @@ "bytes": 2419, "ip": "192.168.0.157", "locality": "internal", - "mac": "b0:34:95:0d:d2:5d", + "mac": "B0-34-95-0D-D2-5D", "packets": 13, "port": 51209 } @@ -1122,7 +1122,7 @@ "packet_delta_count": 10, "protocol_identifier": 6, "source_ipv4_address": "172.217.23.232", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1148,7 +1148,7 @@ "bytes": 5551, "ip": "172.217.23.232", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 10, "port": 443 } @@ -1199,7 +1199,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "107.21.232.174", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 25, "type": "netflow_flow" @@ -1225,7 +1225,7 @@ "bytes": 187, "ip": "107.21.232.174", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 3, "port": 443 } @@ -1276,7 +1276,7 @@ "packet_delta_count": 2, "protocol_identifier": 6, "source_ipv4_address": "192.168.3.178", - "source_mac_address": "dc:ef:ca:4c:da:57", + "source_mac_address": "DC-EF-CA-4C-DA-57", "source_transport_port": 45584, "tcp_control_bits": 17, "type": "netflow_flow" @@ -1302,7 +1302,7 @@ "bytes": 104, "ip": "192.168.3.178", "locality": "internal", - "mac": "dc:ef:ca:4c:da:57", + "mac": "DC-EF-CA-4C-DA-57", "packets": 2, "port": 45584 } @@ -1353,7 +1353,7 @@ "packet_delta_count": 72, "protocol_identifier": 6, "source_ipv4_address": "192.168.2.118", - "source_mac_address": "70:18:8b:5c:c9:b5", + "source_mac_address": "70-18-8B-5C-C9-B5", "source_transport_port": 64233, "tcp_control_bits": 27, "type": "netflow_flow" @@ -1379,7 +1379,7 @@ "bytes": 4050, "ip": "192.168.2.118", "locality": "internal", - "mac": "70:18:8b:5c:c9:b5", + "mac": "70-18-8B-5C-C9-B5", "packets": 72, "port": 64233 } @@ -1430,7 +1430,7 @@ "packet_delta_count": 72, "protocol_identifier": 6, "source_ipv4_address": "95.0.145.242", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 2222, "tcp_control_bits": 27, "type": "netflow_flow" @@ -1456,7 +1456,7 @@ "bytes": 3719, "ip": "95.0.145.242", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 72, "port": 2222 } @@ -1507,7 +1507,7 @@ "packet_delta_count": 16, "protocol_identifier": 6, "source_ipv4_address": "192.168.0.79", - "source_mac_address": "8c:29:37:7a:28:c0", + "source_mac_address": "8C-29-37-7A-28-C0", "source_transport_port": 54275, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1533,7 +1533,7 @@ "bytes": 1402, "ip": "192.168.0.79", "locality": "internal", - "mac": "8c:29:37:7a:28:c0", + "mac": "8C-29-37-7A-28-C0", "packets": 16, "port": 54275 } @@ -1584,7 +1584,7 @@ "packet_delta_count": 17, "protocol_identifier": 6, "source_ipv4_address": "192.168.0.79", - "source_mac_address": "8c:29:37:7a:28:c0", + "source_mac_address": "8C-29-37-7A-28-C0", "source_transport_port": 54276, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1610,7 +1610,7 @@ "bytes": 1538, "ip": "192.168.0.79", "locality": "internal", - "mac": "8c:29:37:7a:28:c0", + "mac": "8C-29-37-7A-28-C0", "packets": 17, "port": 54276 } @@ -1661,7 +1661,7 @@ "packet_delta_count": 14, "protocol_identifier": 6, "source_ipv4_address": "23.5.100.66", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1687,7 +1687,7 @@ "bytes": 13002, "ip": "23.5.100.66", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 14, "port": 443 } @@ -1738,7 +1738,7 @@ "packet_delta_count": 4, "protocol_identifier": 6, "source_ipv4_address": "170.251.180.15", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 24, "type": "netflow_flow" @@ -1764,7 +1764,7 @@ "bytes": 1194, "ip": "170.251.180.15", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 4, "port": 443 } @@ -1815,7 +1815,7 @@ "packet_delta_count": 2, "protocol_identifier": 6, "source_ipv4_address": "192.168.0.61", - "source_mac_address": "90:61:ae:76:e5:e9", + "source_mac_address": "90-61-AE-76-E5-E9", "source_transport_port": 57007, "tcp_control_bits": 24, "type": "netflow_flow" @@ -1841,7 +1841,7 @@ "bytes": 682, "ip": "192.168.0.61", "locality": "internal", - "mac": "90:61:ae:76:e5:e9", + "mac": "90-61-AE-76-E5-E9", "packets": 2, "port": 57007 } @@ -1892,7 +1892,7 @@ "packet_delta_count": 11, "protocol_identifier": 6, "source_ipv4_address": "192.168.3.34", - "source_mac_address": "1c:5c:f2:07:0f:2a", + "source_mac_address": "1C-5C-F2-07-0F-2A", "source_transport_port": 61694, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1918,7 +1918,7 @@ "bytes": 1804, "ip": "192.168.3.34", "locality": "internal", - "mac": "1c:5c:f2:07:0f:2a", + "mac": "1C-5C-F2-07-0F-2A", "packets": 11, "port": 61694 } @@ -1969,7 +1969,7 @@ "packet_delta_count": 9, "protocol_identifier": 6, "source_ipv4_address": "185.60.218.19", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 26, "type": "netflow_flow" @@ -1995,7 +1995,7 @@ "bytes": 4774, "ip": "185.60.218.19", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 9, "port": 443 } @@ -2046,7 +2046,7 @@ "packet_delta_count": 2, "protocol_identifier": 6, "source_ipv4_address": "192.168.3.200", - "source_mac_address": "18:20:32:bb:1d:62", + "source_mac_address": "18-20-32-BB-1D-62", "source_transport_port": 64493, "tcp_control_bits": 24, "type": "netflow_flow" @@ -2072,7 +2072,7 @@ "bytes": 135, "ip": "192.168.3.200", "locality": "internal", - "mac": "18:20:32:bb:1d:62", + "mac": "18-20-32-BB-1D-62", "packets": 2, "port": 64493 } @@ -2123,7 +2123,7 @@ "packet_delta_count": 2, "protocol_identifier": 6, "source_ipv4_address": "185.60.218.15", - "source_mac_address": "00:23:04:18:ef:40", + "source_mac_address": "00-23-04-18-EF-40", "source_transport_port": 443, "tcp_control_bits": 24, "type": "netflow_flow" @@ -2149,7 +2149,7 @@ "bytes": 135, "ip": "185.60.218.15", "locality": "external", - "mac": "00:23:04:18:ef:40", + "mac": "00-23-04-18-EF-40", "packets": 2, "port": 443 } @@ -2200,7 +2200,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "192.168.0.95", - "source_mac_address": "a0:39:f7:4d:49:d5", + "source_mac_address": "A0-39-F7-4D-49-D5", "source_transport_port": 35053, "tcp_control_bits": 24, "type": "netflow_flow" @@ -2226,7 +2226,7 @@ "bytes": 194, "ip": "192.168.0.95", "locality": "internal", - "mac": "a0:39:f7:4d:49:d5", + "mac": "A0-39-F7-4D-49-D5", "packets": 3, "port": 35053 } diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json index f81aba8d536..5dde94af053 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json @@ -8,7 +8,7 @@ "destination": { "ip": "10.30.19.180", "locality": "internal", - "mac": "1c:df:0f:7e:c3:58", + "mac": "1C-DF-0F-7E-C3-58", "port": 2048 }, "event": { @@ -33,7 +33,7 @@ "bgp_destination_as_number": 0, "bgp_source_as_number": 0, "destination_ipv4_address": "10.30.19.180", - "destination_mac_address": "1c:df:0f:7e:c3:58", + "destination_mac_address": "1C-DF-0F-7E-C3-58", "destination_transport_port": 2048, "egress_interface": 0, "exporter": { @@ -57,7 +57,7 @@ "source_ipv4_address": "10.30.18.62", "source_ipv4_prefix": "10.30.18.0", "source_ipv4_prefix_length": 23, - "source_mac_address": "00:50:56:91:56:86", + "source_mac_address": "00-50-56-91-56-86", "source_transport_port": 0, "tcp_source_port": 0, "type": "netflow_flow", @@ -84,7 +84,7 @@ "bytes": 44, "ip": "10.30.18.62", "locality": "internal", - "mac": "00:50:56:91:56:86", + "mac": "00-50-56-91-56-86", "packets": 1, "port": 0 } @@ -99,7 +99,7 @@ "destination": { "ip": "10.30.19.180", "locality": "internal", - "mac": "1c:df:0f:7e:c3:58", + "mac": "1C-DF-0F-7E-C3-58", "port": 161 }, "event": { @@ -124,7 +124,7 @@ "bgp_destination_as_number": 0, "bgp_source_as_number": 0, "destination_ipv4_address": "10.30.19.180", - "destination_mac_address": "1c:df:0f:7e:c3:58", + "destination_mac_address": "1C-DF-0F-7E-C3-58", "destination_transport_port": 161, "egress_interface": 0, "exporter": { @@ -148,7 +148,7 @@ "source_ipv4_address": "10.30.18.62", "source_ipv4_prefix": "10.30.18.0", "source_ipv4_prefix_length": 23, - "source_mac_address": "00:50:56:91:56:86", + "source_mac_address": "00-50-56-91-56-86", "source_transport_port": 34220, "tcp_source_port": 0, "type": "netflow_flow", @@ -175,7 +175,7 @@ "bytes": 106, "ip": "10.30.18.62", "locality": "internal", - "mac": "00:50:56:91:56:86", + "mac": "00-50-56-91-56-86", "packets": 1, "port": 34220 } @@ -190,7 +190,7 @@ "destination": { "ip": "10.30.19.180", "locality": "internal", - "mac": "1c:df:0f:7e:c3:58", + "mac": "1C-DF-0F-7E-C3-58", "port": 2048 }, "event": { @@ -215,7 +215,7 @@ "bgp_destination_as_number": 0, "bgp_source_as_number": 0, "destination_ipv4_address": "10.30.19.180", - "destination_mac_address": "1c:df:0f:7e:c3:58", + "destination_mac_address": "1C-DF-0F-7E-C3-58", "destination_transport_port": 2048, "egress_interface": 0, "exporter": { @@ -239,7 +239,7 @@ "source_ipv4_address": "10.10.172.60", "source_ipv4_prefix": "0.0.0.0", "source_ipv4_prefix_length": 0, - "source_mac_address": "00:18:19:9e:6c:01", + "source_mac_address": "00-18-19-9E-6C-01", "source_transport_port": 0, "tcp_source_port": 0, "type": "netflow_flow", @@ -266,7 +266,7 @@ "bytes": 44, "ip": "10.10.172.60", "locality": "internal", - "mac": "00:18:19:9e:6c:01", + "mac": "00-18-19-9E-6C-01", "packets": 1, "port": 0 } @@ -281,7 +281,7 @@ "destination": { "ip": "10.30.19.180", "locality": "internal", - "mac": "1c:df:0f:7e:c3:58", + "mac": "1C-DF-0F-7E-C3-58", "port": 123 }, "event": { @@ -306,7 +306,7 @@ "bgp_destination_as_number": 0, "bgp_source_as_number": 0, "destination_ipv4_address": "10.30.19.180", - "destination_mac_address": "1c:df:0f:7e:c3:58", + "destination_mac_address": "1C-DF-0F-7E-C3-58", "destination_transport_port": 123, "egress_interface": 0, "exporter": { @@ -330,7 +330,7 @@ "source_ipv4_address": "10.10.172.60", "source_ipv4_prefix": "0.0.0.0", "source_ipv4_prefix_length": 0, - "source_mac_address": "00:18:19:9e:6c:01", + "source_mac_address": "00-18-19-9E-6C-01", "source_transport_port": 123, "tcp_source_port": 0, "type": "netflow_flow", @@ -357,7 +357,7 @@ "bytes": 76, "ip": "10.10.172.60", "locality": "internal", - "mac": "00:18:19:9e:6c:01", + "mac": "00-18-19-9E-6C-01", "packets": 1, "port": 123 } @@ -372,7 +372,7 @@ "destination": { "ip": "10.30.19.180", "locality": "internal", - "mac": "1c:df:0f:7e:c3:58", + "mac": "1C-DF-0F-7E-C3-58", "port": 161 }, "event": { @@ -397,7 +397,7 @@ "bgp_destination_as_number": 0, "bgp_source_as_number": 0, "destination_ipv4_address": "10.30.19.180", - "destination_mac_address": "1c:df:0f:7e:c3:58", + "destination_mac_address": "1C-DF-0F-7E-C3-58", "destination_transport_port": 161, "egress_interface": 0, "exporter": { @@ -421,7 +421,7 @@ "source_ipv4_address": "10.10.172.60", "source_ipv4_prefix": "0.0.0.0", "source_ipv4_prefix_length": 0, - "source_mac_address": "00:18:19:9e:6c:01", + "source_mac_address": "00-18-19-9E-6C-01", "source_transport_port": 45269, "tcp_source_port": 0, "type": "netflow_flow", @@ -448,7 +448,7 @@ "bytes": 2794, "ip": "10.10.172.60", "locality": "internal", - "mac": "00:18:19:9e:6c:01", + "mac": "00-18-19-9E-6C-01", "packets": 36, "port": 45269 } diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json index 2f2c365c63f..7164eaed7b4 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json @@ -6,7 +6,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -37,10 +37,10 @@ "packet_delta_count": 83, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 3320, @@ -56,7 +56,7 @@ "bytes": 3320, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 83 } }, @@ -70,7 +70,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -101,10 +101,10 @@ "packet_delta_count": 83, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 3320, @@ -118,7 +118,7 @@ }, "source": { "bytes": 3320, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 83 } }, @@ -130,7 +130,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -161,10 +161,10 @@ "packet_delta_count": 69, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 7760, @@ -180,7 +180,7 @@ "bytes": 7760, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 69 } }, @@ -194,7 +194,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -225,10 +225,10 @@ "packet_delta_count": 69, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 10229, @@ -242,7 +242,7 @@ }, "source": { "bytes": 10229, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 69 } }, @@ -254,7 +254,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -285,10 +285,10 @@ "packet_delta_count": 1, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 215, @@ -304,7 +304,7 @@ "bytes": 215, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 1 } }, @@ -316,7 +316,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -347,10 +347,10 @@ "packet_delta_count": 225, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 40854, @@ -366,7 +366,7 @@ "bytes": 40854, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 225 } }, @@ -380,7 +380,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -411,10 +411,10 @@ "packet_delta_count": 154, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 35866, @@ -428,7 +428,7 @@ }, "source": { "bytes": 35866, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 154 } }, @@ -440,7 +440,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -471,10 +471,10 @@ "packet_delta_count": 63, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 12279, @@ -490,7 +490,7 @@ "bytes": 12279, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 63 } }, @@ -504,7 +504,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -535,10 +535,10 @@ "packet_delta_count": 61, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 27287, @@ -552,7 +552,7 @@ }, "source": { "bytes": 27287, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 61 } }, @@ -564,7 +564,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -595,10 +595,10 @@ "packet_delta_count": 773, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 147145, @@ -614,7 +614,7 @@ "bytes": 147145, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 773 } }, @@ -628,7 +628,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -659,10 +659,10 @@ "packet_delta_count": 1379, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 1182695, @@ -676,7 +676,7 @@ }, "source": { "bytes": 1182695, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 1379 } }, @@ -688,7 +688,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -719,10 +719,10 @@ "packet_delta_count": 26, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 6777, @@ -738,7 +738,7 @@ "bytes": 6777, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 26 } }, @@ -752,7 +752,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -783,10 +783,10 @@ "packet_delta_count": 26, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 8625, @@ -800,7 +800,7 @@ }, "source": { "bytes": 8625, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 26 } }, @@ -812,7 +812,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -843,10 +843,10 @@ "packet_delta_count": 20434, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 2433001, @@ -862,7 +862,7 @@ "bytes": 2433001, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 20434 } }, @@ -876,7 +876,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -907,10 +907,10 @@ "packet_delta_count": 40726, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 56599680, @@ -924,7 +924,7 @@ }, "source": { "bytes": 56599680, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 40726 } }, @@ -936,7 +936,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -967,10 +967,10 @@ "packet_delta_count": 15, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 1658, @@ -986,7 +986,7 @@ "bytes": 1658, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 15 } }, @@ -1000,7 +1000,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -1031,10 +1031,10 @@ "packet_delta_count": 14, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 950, @@ -1048,7 +1048,7 @@ }, "source": { "bytes": 950, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 14 } }, @@ -1060,7 +1060,7 @@ "Meta": null, "Fields": { "destination": { - "mac": "00:f6:63:cc:80:60" + "mac": "00-F6-63-CC-80-60" }, "event": { "action": "netflow_flow", @@ -1091,10 +1091,10 @@ "packet_delta_count": 16145, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 1495567, @@ -1110,7 +1110,7 @@ "bytes": 1495567, "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51", + "mac": "34-02-86-75-C0-51", "packets": 16145 } }, @@ -1124,7 +1124,7 @@ "destination": { "ip": "192.168.20.121", "locality": "internal", - "mac": "34:02:86:75:c0:51" + "mac": "34-02-86-75-C0-51" }, "event": { "action": "netflow_flow", @@ -1155,10 +1155,10 @@ "packet_delta_count": 53362, "post_ip_diff_serv_code_point": 0, "sta_ipv4_address": "192.168.20.121", - "sta_mac_address": "34:02:86:75:c0:51", + "sta_mac_address": "34-02-86-75-C0-51", "type": "netflow_flow", "wlan_ssid": "Test-env", - "wtp_mac_address": "00:f6:63:cc:80:60" + "wtp_mac_address": "00-F6-63-CC-80-60" }, "network": { "bytes": 80973880, @@ -1172,7 +1172,7 @@ }, "source": { "bytes": 80973880, - "mac": "00:f6:63:cc:80:60", + "mac": "00-F6-63-CC-80-60", "packets": 53362 } }, diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json index 4d2f36f4564..49083084ca1 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json @@ -8,7 +8,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 17232 }, "event": { @@ -31,7 +31,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 17232, "egress_interface": 0, "exporter": { @@ -52,7 +52,7 @@ "packet_delta_count": 2, "protocol_identifier": 17, "source_ipv4_address": "10.1.0.135", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 53, "tcp_control_bits": 0, "type": "netflow_flow", @@ -79,7 +79,7 @@ "bytes": 174, "ip": "10.1.0.135", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 2, "port": 53 } @@ -94,7 +94,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 17232 }, "event": { @@ -117,7 +117,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 17232, "egress_interface": 0, "exporter": { @@ -138,7 +138,7 @@ "packet_delta_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.1.0.136", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 53, "tcp_control_bits": 0, "type": "netflow_flow", @@ -165,7 +165,7 @@ "bytes": 87, "ip": "10.1.0.136", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 1, "port": 53 } @@ -180,7 +180,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 51369 }, "event": { @@ -203,7 +203,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 51369, "egress_interface": 0, "exporter": { @@ -224,7 +224,7 @@ "packet_delta_count": 15, "protocol_identifier": 6, "source_ipv4_address": "10.1.0.232", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 443, "tcp_control_bits": 27, "type": "netflow_flow", @@ -251,7 +251,7 @@ "bytes": 1920, "ip": "10.1.0.232", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 15, "port": 443 } @@ -266,7 +266,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 51370 }, "event": { @@ -289,7 +289,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 51370, "egress_interface": 0, "exporter": { @@ -310,7 +310,7 @@ "packet_delta_count": 8, "protocol_identifier": 6, "source_ipv4_address": "10.1.0.232", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 443, "tcp_control_bits": 27, "type": "netflow_flow", @@ -337,7 +337,7 @@ "bytes": 610, "ip": "10.1.0.232", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 8, "port": 443 } @@ -352,7 +352,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 44006 }, "event": { @@ -375,7 +375,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 44006, "egress_interface": 0, "exporter": { @@ -396,7 +396,7 @@ "packet_delta_count": 21, "protocol_identifier": 6, "source_ipv4_address": "10.5.0.91", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 443, "tcp_control_bits": 31, "type": "netflow_flow", @@ -423,7 +423,7 @@ "bytes": 2420, "ip": "10.5.0.91", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 21, "port": 443 } @@ -438,7 +438,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 33282 }, "event": { @@ -461,7 +461,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 33282, "egress_interface": 0, "exporter": { @@ -482,7 +482,7 @@ "packet_delta_count": 30, "protocol_identifier": 6, "source_ipv4_address": "10.1.0.30", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 443, "tcp_control_bits": 31, "type": "netflow_flow", @@ -509,7 +509,7 @@ "bytes": 10204, "ip": "10.1.0.30", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 30, "port": 443 } @@ -524,7 +524,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 64642 }, "event": { @@ -547,7 +547,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 64642, "egress_interface": 0, "exporter": { @@ -568,7 +568,7 @@ "packet_delta_count": 4, "protocol_identifier": 6, "source_ipv4_address": "10.3.0.100", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 443, "tcp_control_bits": 27, "type": "netflow_flow", @@ -595,7 +595,7 @@ "bytes": 216, "ip": "10.3.0.100", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 4, "port": 443 } @@ -610,7 +610,7 @@ "destination": { "ip": "10.4.0.251", "locality": "internal", - "mac": "44:d9:e7:be:ef:89", + "mac": "44-D9-E7-BE-EF-89", "port": 9497 }, "event": { @@ -633,7 +633,7 @@ "netflow": { "delta_flow_count": 0, "destination_ipv4_address": "10.4.0.251", - "destination_mac_address": "44:d9:e7:be:ef:89", + "destination_mac_address": "44-D9-E7-BE-EF-89", "destination_transport_port": 9497, "egress_interface": 0, "exporter": { @@ -654,7 +654,7 @@ "packet_delta_count": 1, "protocol_identifier": 17, "source_ipv4_address": "10.1.0.135", - "source_mac_address": "06:be:ef:be:ef:4f", + "source_mac_address": "06-BE-EF-BE-EF-4F", "source_transport_port": 53, "tcp_control_bits": 0, "type": "netflow_flow", @@ -681,7 +681,7 @@ "bytes": 152, "ip": "10.1.0.135", "locality": "internal", - "mac": "06:be:ef:be:ef:4f", + "mac": "06-BE-EF-BE-EF-4F", "packets": 1, "port": 53 } @@ -736,8 +736,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 260, "packet_delta_count": 5, - "post_destination_mac_address": "44:d9:e7:be:ef:8e", - "post_source_mac_address": "44:d9:e7:be:ef:22", + "post_destination_mac_address": "44-D9-E7-BE-EF-8E", + "post_source_mac_address": "44-D9-E7-BE-EF-22", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "192.168.1.98", @@ -820,8 +820,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 32, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.4.0.251", @@ -904,8 +904,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 135, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.4.0.251", @@ -988,8 +988,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 135, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.4.0.251", @@ -1072,8 +1072,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 135, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.4.0.251", @@ -1156,8 +1156,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 135, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.4.0.251", @@ -1240,8 +1240,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 135, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.4.0.251", @@ -1324,8 +1324,8 @@ "mpls_label_stack_length": 4, "octet_delta_count": 3668, "packet_delta_count": 21, - "post_destination_mac_address": "44:d9:e7:be:ef:8e", - "post_source_mac_address": "06:be:ef:be:ef:b9", + "post_destination_mac_address": "44-D9-E7-BE-EF-8E", + "post_source_mac_address": "06-BE-EF-BE-EF-B9", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "192.168.1.102", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json index 80d1b0e09c7..1f4315415b2 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json @@ -8,7 +8,7 @@ "destination": { "ip": "193.151.198.166", "locality": "external", - "mac": "00:1b:21:bc:24:dd", + "mac": "00-1B-21-BC-24-DD", "port": 36025 }, "event": { @@ -30,7 +30,7 @@ }, "netflow": { "destination_ipv4_address": "193.151.198.166", - "destination_mac_address": "00:1b:21:bc:24:dd", + "destination_mac_address": "00-1B-21-BC-24-DD", "destination_transport_port": 36025, "egress_interface": 7, "egress_physical_interface": 7, @@ -52,7 +52,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "37.122.1.226", - "source_mac_address": "90:e2:ba:23:09:fc", + "source_mac_address": "90-E2-BA-23-09-FC", "source_transport_port": 27622, "tcp_control_bits": 2, "tcp_options": 4026531840, @@ -79,7 +79,7 @@ "bytes": 156, "ip": "37.122.1.226", "locality": "external", - "mac": "90:e2:ba:23:09:fc", + "mac": "90-E2-BA-23-09-FC", "packets": 3, "port": 27622 } @@ -94,7 +94,7 @@ "destination": { "ip": "193.151.199.69", "locality": "external", - "mac": "00:1b:21:bc:24:dd", + "mac": "00-1B-21-BC-24-DD", "port": 29598 }, "event": { @@ -116,7 +116,7 @@ }, "netflow": { "destination_ipv4_address": "193.151.199.69", - "destination_mac_address": "00:1b:21:bc:24:dd", + "destination_mac_address": "00-1B-21-BC-24-DD", "destination_transport_port": 29598, "egress_interface": 7, "egress_physical_interface": 7, @@ -138,7 +138,7 @@ "packet_delta_count": 1, "protocol_identifier": 6, "source_ipv4_address": "5.141.231.166", - "source_mac_address": "90:e2:ba:23:09:fc", + "source_mac_address": "90-E2-BA-23-09-FC", "source_transport_port": 31178, "tcp_control_bits": 2, "tcp_options": 3489660928, @@ -165,7 +165,7 @@ "bytes": 48, "ip": "5.141.231.166", "locality": "external", - "mac": "90:e2:ba:23:09:fc", + "mac": "90-E2-BA-23-09-FC", "packets": 1, "port": 31178 } @@ -180,7 +180,7 @@ "destination": { "ip": "212.224.113.74", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 443 }, "event": { @@ -202,7 +202,7 @@ }, "netflow": { "destination_ipv4_address": "212.224.113.74", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 443, "egress_interface": 7, "egress_physical_interface": 7, @@ -224,7 +224,7 @@ "packet_delta_count": 11, "protocol_identifier": 6, "source_ipv4_address": "10.233.128.4", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 53688, "tcp_control_bits": 211, "tcp_options": 4043309057, @@ -251,7 +251,7 @@ "bytes": 584, "ip": "10.233.128.4", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 11, "port": 53688 } @@ -266,7 +266,7 @@ "destination": { "ip": "10.236.8.4", "locality": "internal", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 51549 }, "event": { @@ -288,7 +288,7 @@ }, "netflow": { "destination_ipv4_address": "10.236.8.4", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 51549, "egress_interface": 8, "egress_physical_interface": 8, @@ -310,7 +310,7 @@ "packet_delta_count": 4, "protocol_identifier": 6, "source_ipv4_address": "193.151.192.46", - "source_mac_address": "00:1a:4a:16:01:81", + "source_mac_address": "00-1A-4A-16-01-81", "source_transport_port": 80, "tcp_control_bits": 27, "tcp_options": 4043309056, @@ -337,7 +337,7 @@ "bytes": 577, "ip": "193.151.192.46", "locality": "external", - "mac": "00:1a:4a:16:01:81", + "mac": "00-1A-4A-16-01-81", "packets": 4, "port": 80 } @@ -352,7 +352,7 @@ "destination": { "ip": "62.221.115.205", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 1024 }, "event": { @@ -374,7 +374,7 @@ }, "netflow": { "destination_ipv4_address": "62.221.115.205", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 1024, "egress_interface": 7, "egress_physical_interface": 7, @@ -396,7 +396,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "10.235.197.6", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 57505, "tcp_control_bits": 2, "tcp_options": 4026531840, @@ -423,7 +423,7 @@ "bytes": 152, "ip": "10.235.197.6", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 3, "port": 57505 } @@ -438,7 +438,7 @@ "destination": { "ip": "37.146.125.64", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 3237 }, "event": { @@ -460,7 +460,7 @@ }, "netflow": { "destination_ipv4_address": "37.146.125.64", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 3237, "egress_interface": 7, "egress_physical_interface": 7, @@ -482,7 +482,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "10.236.31.7", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 61471, "tcp_control_bits": 2, "tcp_options": 4026531840, @@ -509,7 +509,7 @@ "bytes": 152, "ip": "10.236.31.7", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 3, "port": 61471 } @@ -524,7 +524,7 @@ "destination": { "ip": "52.198.214.72", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 443 }, "event": { @@ -546,7 +546,7 @@ }, "netflow": { "destination_ipv4_address": "52.198.214.72", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 443, "egress_interface": 7, "egress_physical_interface": 7, @@ -568,7 +568,7 @@ "packet_delta_count": 15, "protocol_identifier": 6, "source_ipv4_address": "10.233.151.8", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 58044, "tcp_control_bits": 31, "tcp_options": 4177526784, @@ -595,7 +595,7 @@ "bytes": 1809, "ip": "10.233.151.8", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 15, "port": 58044 } @@ -610,7 +610,7 @@ "destination": { "ip": "64.233.161.188", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 5228 }, "event": { @@ -632,7 +632,7 @@ }, "netflow": { "destination_ipv4_address": "64.233.161.188", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 5228, "egress_interface": 7, "egress_physical_interface": 7, @@ -654,7 +654,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "10.234.22.4", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 60583, "tcp_control_bits": 24, "tcp_options": 2164260864, @@ -681,7 +681,7 @@ "bytes": 234, "ip": "10.234.22.4", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 3, "port": 60583 } @@ -696,7 +696,7 @@ "destination": { "ip": "185.209.20.240", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 80 }, "event": { @@ -718,7 +718,7 @@ }, "netflow": { "destination_ipv4_address": "185.209.20.240", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 80, "egress_interface": 7, "egress_physical_interface": 7, @@ -740,7 +740,7 @@ "packet_delta_count": 22, "protocol_identifier": 6, "source_ipv4_address": "10.233.36.7", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 51399, "tcp_control_bits": 27, "tcp_options": 4043309056, @@ -767,7 +767,7 @@ "bytes": 1681, "ip": "10.233.36.7", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 22, "port": 51399 } @@ -782,7 +782,7 @@ "destination": { "ip": "84.39.245.175", "locality": "external", - "mac": "00:1b:21:bc:24:dc", + "mac": "00-1B-21-BC-24-DC", "port": 18580 }, "event": { @@ -804,7 +804,7 @@ }, "netflow": { "destination_ipv4_address": "84.39.245.175", - "destination_mac_address": "00:1b:21:bc:24:dc", + "destination_mac_address": "00-1B-21-BC-24-DC", "destination_transport_port": 18580, "egress_interface": 7, "egress_physical_interface": 7, @@ -826,7 +826,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "10.233.200.7", - "source_mac_address": "00:04:96:97:b8:cd", + "source_mac_address": "00-04-96-97-B8-CD", "source_transport_port": 61820, "tcp_control_bits": 2, "tcp_options": 4026531840, @@ -853,7 +853,7 @@ "bytes": 152, "ip": "10.233.200.7", "locality": "internal", - "mac": "00:04:96:97:b8:cd", + "mac": "00-04-96-97-B8-CD", "packets": 3, "port": 61820 } @@ -868,7 +868,7 @@ "destination": { "ip": "10.232.8.45", "locality": "internal", - "mac": "00:1b:21:bc:24:dd", + "mac": "00-1B-21-BC-24-DD", "port": 56257 }, "event": { @@ -890,7 +890,7 @@ }, "netflow": { "destination_ipv4_address": "10.232.8.45", - "destination_mac_address": "00:1b:21:bc:24:dd", + "destination_mac_address": "00-1B-21-BC-24-DD", "destination_transport_port": 56257, "egress_interface": 8, "egress_physical_interface": 8, @@ -912,7 +912,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "23.43.139.27", - "source_mac_address": "90:e2:ba:23:09:fc", + "source_mac_address": "90-E2-BA-23-09-FC", "source_transport_port": 80, "tcp_control_bits": 26, "tcp_options": 4026531840, @@ -939,7 +939,7 @@ "bytes": 1866, "ip": "23.43.139.27", "locality": "external", - "mac": "90:e2:ba:23:09:fc", + "mac": "90-E2-BA-23-09-FC", "packets": 3, "port": 80 } @@ -954,7 +954,7 @@ "destination": { "ip": "10.233.150.21", "locality": "internal", - "mac": "00:1b:21:bc:24:dd", + "mac": "00-1B-21-BC-24-DD", "port": 38164 }, "event": { @@ -976,7 +976,7 @@ }, "netflow": { "destination_ipv4_address": "10.233.150.21", - "destination_mac_address": "00:1b:21:bc:24:dd", + "destination_mac_address": "00-1B-21-BC-24-DD", "destination_transport_port": 38164, "egress_interface": 8, "egress_physical_interface": 8, @@ -998,7 +998,7 @@ "packet_delta_count": 3, "protocol_identifier": 6, "source_ipv4_address": "2.17.140.47", - "source_mac_address": "90:e2:ba:23:09:fc", + "source_mac_address": "90-E2-BA-23-09-FC", "source_transport_port": 443, "tcp_control_bits": 25, "tcp_options": 2164260864, @@ -1025,7 +1025,7 @@ "bytes": 187, "ip": "2.17.140.47", "locality": "external", - "mac": "90:e2:ba:23:09:fc", + "mac": "90-E2-BA-23-09-FC", "packets": 3, "port": 443 } diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json index e2aee36c229..6135a7e669e 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json @@ -43,7 +43,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 22 }, "event": { @@ -62,7 +62,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 22, "exporter": { "address": "192.0.2.1:4444", @@ -73,7 +73,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 65058, "type": "netflow_flow" }, @@ -95,7 +95,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 65058 } }, @@ -109,7 +109,7 @@ "destination": { "ip": "172.16.32.100", "locality": "internal", - "mac": "00:0c:29:8d:af:c3", + "mac": "00-0C-29-8D-AF-C3", "port": 123 }, "event": { @@ -128,7 +128,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.100", - "destination_mac_address": "00:0c:29:8d:af:c3", + "destination_mac_address": "00-0C-29-8D-AF-C3", "destination_transport_port": 123, "exporter": { "address": "192.0.2.1:4444", @@ -139,7 +139,7 @@ }, "protocol_identifier": 17, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 123, "type": "netflow_flow" }, @@ -161,7 +161,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 123 } }, @@ -175,7 +175,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 123 }, "event": { @@ -194,7 +194,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 123, "exporter": { "address": "192.0.2.1:4444", @@ -205,7 +205,7 @@ }, "protocol_identifier": 17, "source_ipv4_address": "172.16.32.100", - "source_mac_address": "00:0c:29:8d:af:c3", + "source_mac_address": "00-0C-29-8D-AF-C3", "source_transport_port": 123, "type": "netflow_flow" }, @@ -227,7 +227,7 @@ "source": { "ip": "172.16.32.100", "locality": "internal", - "mac": "00:0c:29:8d:af:c3", + "mac": "00-0C-29-8D-AF-C3", "port": 123 } }, @@ -241,7 +241,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 80 }, "event": { @@ -260,7 +260,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 80, "exporter": { "address": "192.0.2.1:4444", @@ -271,7 +271,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59157, "type": "netflow_flow" }, @@ -293,7 +293,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59157 } }, @@ -307,7 +307,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59157 }, "event": { @@ -326,7 +326,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59157, "exporter": { "address": "192.0.2.1:4444", @@ -337,7 +337,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 80, "type": "netflow_flow" }, @@ -359,7 +359,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 80 } }, @@ -373,7 +373,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 443 }, "event": { @@ -392,7 +392,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 443, "exporter": { "address": "192.0.2.1:4444", @@ -403,7 +403,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59158, "type": "netflow_flow" }, @@ -425,7 +425,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59158 } }, @@ -439,7 +439,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59158 }, "event": { @@ -458,7 +458,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59158, "exporter": { "address": "192.0.2.1:4444", @@ -469,7 +469,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 443, "type": "netflow_flow" }, @@ -491,7 +491,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 443 } }, @@ -505,7 +505,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 139 }, "event": { @@ -524,7 +524,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 139, "exporter": { "address": "192.0.2.1:4444", @@ -535,7 +535,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59159, "type": "netflow_flow" }, @@ -557,7 +557,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59159 } }, @@ -571,7 +571,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59159 }, "event": { @@ -590,7 +590,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59159, "exporter": { "address": "192.0.2.1:4444", @@ -601,7 +601,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 139, "type": "netflow_flow" }, @@ -623,7 +623,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 139 } }, @@ -637,7 +637,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 23 }, "event": { @@ -656,7 +656,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 23, "exporter": { "address": "192.0.2.1:4444", @@ -667,7 +667,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59160, "type": "netflow_flow" }, @@ -689,7 +689,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59160 } }, @@ -703,7 +703,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59160 }, "event": { @@ -722,7 +722,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59160, "exporter": { "address": "192.0.2.1:4444", @@ -733,7 +733,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 23, "type": "netflow_flow" }, @@ -755,7 +755,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 23 } }, @@ -769,7 +769,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 995 }, "event": { @@ -788,7 +788,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 995, "exporter": { "address": "192.0.2.1:4444", @@ -799,7 +799,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59161, "type": "netflow_flow" }, @@ -821,7 +821,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59161 } }, @@ -835,7 +835,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59161 }, "event": { @@ -854,7 +854,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59161, "exporter": { "address": "192.0.2.1:4444", @@ -865,7 +865,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 995, "type": "netflow_flow" }, @@ -887,7 +887,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 995 } }, @@ -901,7 +901,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 443 }, "event": { @@ -920,7 +920,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 443, "exporter": { "address": "192.0.2.1:4444", @@ -931,7 +931,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59162, "type": "netflow_flow" }, @@ -953,7 +953,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59162 } }, @@ -967,7 +967,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59162 }, "event": { @@ -986,7 +986,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59162, "exporter": { "address": "192.0.2.1:4444", @@ -997,7 +997,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 443, "type": "netflow_flow" }, @@ -1019,7 +1019,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 443 } }, @@ -1033,7 +1033,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 135 }, "event": { @@ -1052,7 +1052,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 135, "exporter": { "address": "192.0.2.1:4444", @@ -1063,7 +1063,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59163, "type": "netflow_flow" }, @@ -1085,7 +1085,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59163 } }, @@ -1099,7 +1099,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59163 }, "event": { @@ -1118,7 +1118,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59163, "exporter": { "address": "192.0.2.1:4444", @@ -1129,7 +1129,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 135, "type": "netflow_flow" }, @@ -1151,7 +1151,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 135 } }, @@ -1165,7 +1165,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 110 }, "event": { @@ -1184,7 +1184,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 110, "exporter": { "address": "192.0.2.1:4444", @@ -1195,7 +1195,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59164, "type": "netflow_flow" }, @@ -1217,7 +1217,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59164 } }, @@ -1231,7 +1231,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59164 }, "event": { @@ -1250,7 +1250,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59164, "exporter": { "address": "192.0.2.1:4444", @@ -1261,7 +1261,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 110, "type": "netflow_flow" }, @@ -1283,7 +1283,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 110 } }, @@ -1297,7 +1297,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 111 }, "event": { @@ -1316,7 +1316,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 111, "exporter": { "address": "192.0.2.1:4444", @@ -1327,7 +1327,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59165, "type": "netflow_flow" }, @@ -1349,7 +1349,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59165 } }, @@ -1363,7 +1363,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59165 }, "event": { @@ -1382,7 +1382,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59165, "exporter": { "address": "192.0.2.1:4444", @@ -1393,7 +1393,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 111, "type": "netflow_flow" }, @@ -1415,7 +1415,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 111 } }, @@ -1429,7 +1429,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 143 }, "event": { @@ -1448,7 +1448,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 143, "exporter": { "address": "192.0.2.1:4444", @@ -1459,7 +1459,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59166, "type": "netflow_flow" }, @@ -1481,7 +1481,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59166 } }, @@ -1495,7 +1495,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59166 }, "event": { @@ -1514,7 +1514,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59166, "exporter": { "address": "192.0.2.1:4444", @@ -1525,7 +1525,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 143, "type": "netflow_flow" }, @@ -1547,7 +1547,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 143 } }, @@ -1561,7 +1561,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 3389 }, "event": { @@ -1580,7 +1580,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 3389, "exporter": { "address": "192.0.2.1:4444", @@ -1591,7 +1591,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59167, "type": "netflow_flow" }, @@ -1613,7 +1613,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59167 } }, @@ -1627,7 +1627,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59167 }, "event": { @@ -1646,7 +1646,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59167, "exporter": { "address": "192.0.2.1:4444", @@ -1657,7 +1657,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 3389, "type": "netflow_flow" }, @@ -1679,7 +1679,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 3389 } }, @@ -1693,7 +1693,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 80 }, "event": { @@ -1712,7 +1712,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 80, "exporter": { "address": "192.0.2.1:4444", @@ -1723,7 +1723,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59168, "type": "netflow_flow" }, @@ -1745,7 +1745,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59168 } }, @@ -1759,7 +1759,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59168 }, "event": { @@ -1778,7 +1778,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59168, "exporter": { "address": "192.0.2.1:4444", @@ -1789,7 +1789,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 80, "type": "netflow_flow" }, @@ -1811,7 +1811,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 80 } }, @@ -1825,7 +1825,7 @@ "destination": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 25 }, "event": { @@ -1844,7 +1844,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.201", - "destination_mac_address": "00:0c:29:70:86:09", + "destination_mac_address": "00-0C-29-70-86-09", "destination_transport_port": 25, "exporter": { "address": "192.0.2.1:4444", @@ -1855,7 +1855,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.1", - "source_mac_address": "00:50:56:c0:00:01", + "source_mac_address": "00-50-56-C0-00-01", "source_transport_port": 59169, "type": "netflow_flow" }, @@ -1877,7 +1877,7 @@ "source": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59169 } }, @@ -1891,7 +1891,7 @@ "destination": { "ip": "172.16.32.1", "locality": "internal", - "mac": "00:50:56:c0:00:01", + "mac": "00-50-56-C0-00-01", "port": 59169 }, "event": { @@ -1910,7 +1910,7 @@ }, "netflow": { "destination_ipv4_address": "172.16.32.1", - "destination_mac_address": "00:50:56:c0:00:01", + "destination_mac_address": "00-50-56-C0-00-01", "destination_transport_port": 59169, "exporter": { "address": "192.0.2.1:4444", @@ -1921,7 +1921,7 @@ }, "protocol_identifier": 6, "source_ipv4_address": "172.16.32.201", - "source_mac_address": "00:0c:29:70:86:09", + "source_mac_address": "00-0C-29-70-86-09", "source_transport_port": 25, "type": "netflow_flow" }, @@ -1943,7 +1943,7 @@ "source": { "ip": "172.16.32.201", "locality": "internal", - "mac": "00:0c:29:70:86:09", + "mac": "00-0C-29-70-86-09", "port": 25 } }, diff --git a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json index 39779cda488..f3e6e23ff7b 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json @@ -48,8 +48,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 421, "packet_delta_count": 6, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.5.2", @@ -132,8 +132,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 7621, "packet_delta_count": 131, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.93", @@ -216,8 +216,8 @@ "mpls_label_stack_length": 3, "octet_delta_count": 95, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "10.100.4.1", @@ -300,8 +300,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 3162, "packet_delta_count": 30, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.93", @@ -384,8 +384,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 2711, "packet_delta_count": 13, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.80", @@ -468,8 +468,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 20855, "packet_delta_count": 346, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.93", @@ -552,8 +552,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 7495, "packet_delta_count": 129, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.93", @@ -636,8 +636,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 7049, "packet_delta_count": 119, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.93", @@ -720,8 +720,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 1348, "packet_delta_count": 13, - "post_destination_mac_address": "04:18:d6:f0:81:b5", - "post_source_mac_address": "00:11:bb:7f:20:42", + "post_destination_mac_address": "04-18-D6-F0-81-B5", + "post_source_mac_address": "00-11-BB-7F-20-42", "post_vlan_id": 0, "protocol_identifier": 6, "source_ipv4_address": "10.100.6.93", @@ -804,8 +804,8 @@ "mpls_label_stack_length": 2, "octet_delta_count": 82, "packet_delta_count": 1, - "post_destination_mac_address": "00:00:00:00:00:00", - "post_source_mac_address": "00:00:00:00:00:00", + "post_destination_mac_address": "00-00-00-00-00-00", + "post_source_mac_address": "00-00-00-00-00-00", "post_vlan_id": 0, "protocol_identifier": 17, "source_ipv4_address": "192.168.1.4", diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index c2a46c88090..b432bb49410 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -100,10 +100,10 @@ processors: field: "json.eventCategory" target_field: "aws.cloudtrail.event_category" ignore_failure: true - - rename: - field: "json.awsRegion" - target_field: "cloud.region" - ignore_failure: true + - set: + field: "cloud.region" + copy_from: "json.awsRegion" + ignore_empty_value: true - rename: field: "json.sourceIPAddress" target_field: "source.address" @@ -608,7 +608,7 @@ processors: target_field: "cloud.account.id" ignore_failure: true - rename: - field: "json.previousDigestS3Object" + field: "json.digestS3Object" target_field: "file.path" ignore_failure: true - rename: diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json index 2bb31cd7284..f57a61d3c29 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json @@ -114,7 +114,7 @@ "event.original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", "event.type": "info", "file.hash.sha256": "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987", - "file.path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", + "file.path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz", "fileset.name": "cloudtrail", "input.type": "log", "log.offset": 0, diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-110da36b-6544-4507-9ae5-591d44bd2333.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-110da36b-6544-4507-9ae5-591d44bd2333.json new file mode 100644 index 00000000000..d309b168cb9 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-110da36b-6544-4507-9ae5-591d44bd2333.json @@ -0,0 +1,433 @@ +{ + "attributes": { + "description": "Operating system activity from endpoints via ArcSight", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Count": "#64B0C8", + "Destination User Names": "#E24D42", + "Event Types": "#EF843C" + }, + "legendOpen": true + } + }, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 28 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "4", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "4", + "panelRefName": "panel_4", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 55k": "rgb(255,255,204)", + "110k - 165k": "rgb(254,225,135)", + "165k - 220k": "rgb(254,201,101)", + "220k - 275k": "rgb(254,171,73)", + "275k - 330k": "rgb(253,141,60)", + "330k - 385k": "rgb(252,91,46)", + "385k - 440k": "rgb(237,47,34)", + "440k - 495k": "rgb(212,16,32)", + "495k - 550k": "rgb(176,0,38)", + "55k - 110k": "rgb(255,241,170)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 12, + "i": "5", + "w": 24, + "x": 24, + "y": 28 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "7", + "w": 48, + "x": 0, + "y": 20 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#447EBC", + "/Failure": "#E24D42", + "/Success": "#7EB26D" + } + } + }, + "gridData": { + "h": 12, + "i": "8", + "w": 24, + "x": 24, + "y": 52 + }, + "panelIndex": "8", + "panelRefName": "panel_8", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 24, + "i": "9", + "w": 24, + "x": 0, + "y": 40 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "10", + "w": 24, + "x": 24, + "y": 40 + }, + "panelIndex": "10", + "panelRefName": "panel_10", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "11", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "12", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Destination Users": "#E24D42", + "Event Count": "#64B0C8" + } + } + }, + "gridData": { + "h": 8, + "i": "13", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 20, + "i": "14", + "w": 16, + "x": 32, + "y": 64 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 24, + "i": "15", + "w": 16, + "x": 32, + "y": 84 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "16", + "w": 32, + "x": 0, + "y": 80 + }, + "panelIndex": "16", + "panelRefName": "panel_16", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "17", + "w": 32, + "x": 0, + "y": 100 + }, + "panelIndex": "17", + "panelRefName": "panel_17", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "18", + "w": 32, + "x": 0, + "y": 64 + }, + "panelIndex": "18", + "panelRefName": "panel_18", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "19", + "w": 32, + "x": 0, + "y": 92 + }, + "panelIndex": "19", + "panelRefName": "panel_19", + "type": "visualization", + "version": "7.3.0" + } + ], + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] ArcSight — Endpoint OS Activity Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-110da36b-6544-4507-9ae5-591d44bd2333", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-81cc567b-7ef9-44f8-9fc6-9fd98f8d8a3d", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-f5fde1d9-50ff-4a55-aa6a-724a8d6c05b2", + "name": "4:panel_4", + "type": "visualization" + }, + { + "id": "cef-09047d0c-900c-4949-9cf6-122154cc203f", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-c77b16c9-031b-45ec-af29-b8f116c610ba", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-dc5d284d-a136-4207-9557-657f0972d534", + "name": "8:panel_8", + "type": "visualization" + }, + { + "id": "cef-18a1d5b4-8f9e-43b6-a6e7-d9f24db85136", + "name": "9:panel_9", + "type": "visualization" + }, + { + "id": "cef-6d82b7f1-5400-43b4-91c0-9f27952d571f", + "name": "10:panel_10", + "type": "visualization" + }, + { + "id": "cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-3ab5a5fa-3775-49d3-892b-4362531971cd", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-97a3e327-9b8b-4a1b-91ad-6c93c90d0193", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-e386b2cc-4e81-4e16-bbfa-65cf583dd247", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-08dd95b3-aa74-4271-9c53-30e106fd5ea6", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "cef-0b7d6680-54e6-4afd-9748-a686ad4d72f0", + "name": "16:panel_16", + "type": "visualization" + }, + { + "id": "cef-02b3c5cc-240f-4ca5-b49f-56e2f1cc83fc", + "name": "17:panel_17", + "type": "visualization" + }, + { + "id": "cef-a29c8241-2598-4964-b800-0e298e9d2ad8", + "name": "18:panel_18", + "type": "visualization" + }, + { + "id": "cef-a09db928-f867-46cf-85f9-1d0a29afa903", + "name": "19:panel_19", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-1ef1282f-382b-4bdd-9cb1-810b8f083e74.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-1ef1282f-382b-4bdd-9cb1-810b8f083e74.json new file mode 100644 index 00000000000..4ae9e6be1a5 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-1ef1282f-382b-4bdd-9cb1-810b8f083e74.json @@ -0,0 +1,609 @@ +{ + "attributes": { + "description": "Summary of ArcSight endpoint event data", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "1", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "gridData": { + "h": 12, + "i": "2", + "w": 24, + "x": 24, + "y": 32 + }, + "panelIndex": "2", + "panelRefName": "panel_2", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 32 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "6", + "w": 24, + "x": 24, + "y": 44 + }, + "panelIndex": "6", + "panelRefName": "panel_6", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "7", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "gridData": { + "h": 12, + "i": "8", + "w": 24, + "x": 0, + "y": 44 + }, + "panelIndex": "8", + "panelRefName": "panel_8", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "columns": [ + "cef.extensions.categoryDeviceGroup", + "cef.extensions.categoryTechnique", + "cef.extensions.categoryOutcome", + "cef.extensions.categorySignificance", + "cef.extensions.categoryObject", + "cef.extensions.categoryBehavior", + "cef.extensions.categoryDeviceType" + ], + "enhancements": {}, + "sort": [ + "@timestamp", + "desc" + ] + }, + "gridData": { + "h": 20, + "i": "9", + "w": 48, + "x": 0, + "y": 76 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "search", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "10", + "w": 24, + "x": 24, + "y": 56 + }, + "panelIndex": "10", + "panelRefName": "panel_10", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Anti-Virus": "#EAB839", + "Database": "#629E51", + "Host-based IDS/IPS": "#E0752D", + "Operating System": "#BF1B00", + "Security Mangement": "#64B0C8" + } + } + }, + "gridData": { + "h": 12, + "i": "11", + "w": 24, + "x": 0, + "y": 20 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 20, + "i": "12", + "w": 24, + "x": 0, + "y": 56 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Informational": "#7EB26D", + "/Informational/Warning": "#EF843C", + "/Success": "#629E51", + "Anti-Virus": "#EAB839", + "Database": "#629E51", + "Host-based IDS/IPS": "#E0752D", + "Log Consolidator": "#E0F9D7", + "Operating System": "#BF1B00", + "Recon": "#BF1B00", + "Security Mangement": "#64B0C8" + } + } + }, + "gridData": { + "h": 12, + "i": "14", + "w": 24, + "x": 24, + "y": 20 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "15", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "de084257-24da-4ea9-922e-a2d7565ebcd6", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "741ceaa6-5b51-4959-9935-c5961b12f539", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Destination Locations by Event — ArcSight [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "destination.geo.location", + "id": "ba850a09-c635-4855-b68b-de16dd200d6f", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Destination Locations by Event — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 12, + "i": "c9fd3ece-2bef-4cdc-9f83-ed689b35a17a", + "w": 24, + "x": 24, + "y": 64 + }, + "panelIndex": "c9fd3ece-2bef-4cdc-9f83-ed689b35a17a", + "type": "map", + "version": "8.3.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] ArcSight — Endpoint Overview Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1ef1282f-382b-4bdd-9cb1-810b8f083e74", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-66c201f1-2ffb-4eea-b408-a59315998364", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-b85d3a62-8dc5-45f8-a561-b0fb5d2d7a5e", + "name": "2:panel_2", + "type": "visualization" + }, + { + "id": "cef-01c35a60-419a-4703-ba46-ca6a8a6d1acf", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-71a210f8-75b5-412b-b5e7-730d8976988d", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-0af23787-8d02-4028-9f41-f32398fefbd0", + "name": "6:panel_6", + "type": "visualization" + }, + { + "id": "cef-4410b326-f66e-4dc6-88f7-8ea234087581", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-c0cf5f53-3ddc-463f-aee7-71d7219a36c5", + "name": "8:panel_8", + "type": "visualization" + }, + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "9:panel_9", + "type": "search" + }, + { + "id": "cef-f3b8b32b-e4a0-4738-b9d1-ca878f3530b1", + "name": "10:panel_10", + "type": "visualization" + }, + { + "id": "cef-42d96ddb-1e83-49fc-91e6-fb189e0b2661", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-4fe11728-f553-403f-8dda-952e28acb607", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-46f5dcfe-2d61-4022-9cbb-2b044135fc15", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "filebeat-*", + "name": "c9fd3ece-2bef-4cdc-9f83-ed689b35a17a:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-244684fd-7be2-49fb-a817-e51cec267bf1.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-244684fd-7be2-49fb-a817-e51cec267bf1.json new file mode 100644 index 00000000000..ae21d0458d5 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-244684fd-7be2-49fb-a817-e51cec267bf1.json @@ -0,0 +1,754 @@ +{ + "attributes": { + "description": "Overview of Microsoft DNS activity", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "1", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "3", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 18k": "rgb(247,251,255)", + "108k - 126k": "rgb(74,152,201)", + "126k - 144k": "rgb(46,126,188)", + "144k - 162k": "rgb(23,100,171)", + "162k - 180k": "rgb(8,74,145)", + "18k - 36k": "rgb(227,238,249)", + "36k - 54k": "rgb(208,225,242)", + "54k - 72k": "rgb(182,212,233)", + "72k - 90k": "rgb(148,196,223)", + "90k - 108k": "rgb(107,174,214)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 16, + "i": "5", + "w": 24, + "x": 0, + "y": 32 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "6", + "w": 48, + "x": 0, + "y": 48 + }, + "panelIndex": "6", + "panelRefName": "panel_6", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "7", + "w": 24, + "x": 24, + "y": 32 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "9", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 16, + "i": "11", + "w": 24, + "x": 24, + "y": 56 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "12", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 16, + "i": "13", + "w": 24, + "x": 0, + "y": 56 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "14", + "w": 24, + "x": 0, + "y": 20 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "15", + "w": 24, + "x": 24, + "y": 20 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "56b3b288-a0f1-416d-9d40-96a37c8484fd", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "d50cbece-4556-4421-bb06-fb015bfe7baa", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Sources by Events [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "source.geo.location", + "id": "555cbeac-b098-4946-9498-6b700e745e8a", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Sources by Events [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 12, + "i": "3cf2118b-5231-49f5-b685-0ff0e1f52c32", + "w": 24, + "x": 0, + "y": 72 + }, + "panelIndex": "3cf2118b-5231-49f5-b685-0ff0e1f52c32", + "type": "map", + "version": "8.3.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "5231e15c-d374-46ca-9553-3308d723ded3", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "8cdaae20-5dcc-4930-b105-802fc344fcb6", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Destinations by Events [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "destination.geo.location", + "id": "88700fdc-3a96-46b8-b51f-3839111eb6ec", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Destinations by Events [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 12, + "i": "07f92eca-2078-4aa6-8373-d27ca33595d6", + "w": 24, + "x": 24, + "y": 72 + }, + "panelIndex": "07f92eca-2078-4aa6-8373-d27ca33595d6", + "type": "map", + "version": "8.3.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] Microsoft DNS Overview", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-244684fd-7be2-49fb-a817-e51cec267bf1", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-c3b99882-cd00-4bc1-a64f-d7d7ac3ad292", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-1013a3cb-ca15-459e-b1a6-45f110958b5c", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-7cd4866d-4085-44d1-ad12-45559570a176", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-119bcab5-9bb0-4644-8dfa-15ff8848c0a2", + "name": "6:panel_6", + "type": "visualization" + }, + { + "id": "cef-8613b170-e504-4a37-b723-baeb94470380", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-1e9875ef-0cfd-40d1-b261-e02838274a7c", + "name": "9:panel_9", + "type": "visualization" + }, + { + "id": "cef-95877a19-0c82-4194-ad67-b311c458c6f4", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-6a5a6bfa-8f61-418e-8fc4-0c5d8ddb41e1", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-fb21024b-49db-405d-85fa-ab7fd43e3753", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-bfc6d482-4f8e-40f8-b4ee-10ba19675d42", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "filebeat-*", + "name": "3cf2118b-5231-49f5-b685-0ff0e1f52c32:layer_1_source_index_pattern", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "07f92eca-2078-4aa6-8373-d27ca33595d6:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-2745d8ff-e261-428b-a406-15ee77c8a3dd.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-2745d8ff-e261-428b-a406-15ee77c8a3dd.json new file mode 100644 index 00000000000..f18bb5efe79 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-2745d8ff-e261-428b-a406-15ee77c8a3dd.json @@ -0,0 +1,310 @@ +{ + "attributes": { + "description": "Suspicious network activity overview", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Destination Addresses": "#E0752D", + "Destination Ports": "#E24D42" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 12, + "i": "1", + "w": 48, + "x": 0, + "y": 28 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 12, + "i": "2", + "w": 16, + "x": 0, + "y": 40 + }, + "panelIndex": "2", + "panelRefName": "panel_2", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 12, + "i": "3", + "w": 16, + "x": 16, + "y": 40 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 20 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "11", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "12", + "w": 24, + "x": 0, + "y": 52 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "13", + "w": 24, + "x": 24, + "y": 52 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "14", + "w": 16, + "x": 32, + "y": 40 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "15", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "16", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "16", + "panelRefName": "panel_16", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 50": "rgb(255,255,204)", + "100 - 200": "rgb(253,141,60)", + "200 - 300": "rgb(227,27,28)", + "300 - 400": "rgb(128,0,38)", + "50 - 100": "rgb(254,217,118)" + } + } + }, + "gridData": { + "h": 8, + "i": "17", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "17", + "panelRefName": "panel_17", + "type": "visualization", + "version": "7.3.0" + } + ], + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] Network Suspicious Activity Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-2745d8ff-e261-428b-a406-15ee77c8a3dd", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-d1332acb-7ace-4ee6-96d7-3a154a0405c7", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-a35b83af-0f72-4825-922f-76a71022c2fb", + "name": "2:panel_2", + "type": "visualization" + }, + { + "id": "cef-9e73269e-bb91-45d0-9b5f-2c2ed32c451f", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-53b315dc-f7f1-444b-8556-8f1cd5dae16a", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-1f4fa9d2-128f-4fca-bb20-f3fca2be6961", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-8f26b7a6-7ae0-4a01-9c0e-b7a0e0302fd7", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-e9d591ca-fc0c-40fd-b3b6-391df6ace611", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-1a9f1963-aa1c-418f-8c15-74dbbcd87758", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "cef-09816af7-d246-4880-b4c8-1ace7ea35b3b", + "name": "16:panel_16", + "type": "visualization" + }, + { + "id": "cef-4e4c17aa-f137-4cfd-80dd-7519f72bb7af", + "name": "17:panel_17", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-346ac50b-0900-464d-aad6-007756a4e215.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-346ac50b-0900-464d-aad6-007756a4e215.json new file mode 100644 index 00000000000..54228b1b236 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-346ac50b-0900-464d-aad6-007756a4e215.json @@ -0,0 +1,860 @@ +{ + "attributes": { + "description": "Network data overview via ArcSight", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "1", + "w": 48, + "x": 0, + "y": 44 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "2", + "w": 48, + "x": 0, + "y": 68 + }, + "panelIndex": "2", + "panelRefName": "panel_2", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "6", + "w": 48, + "x": 0, + "y": 60 + }, + "panelIndex": "6", + "panelRefName": "panel_6", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 8, + "i": "7", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "gridData": { + "h": 12, + "i": "9", + "w": 16, + "x": 0, + "y": 20 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "gridData": { + "h": 12, + "i": "11", + "w": 16, + "x": 16, + "y": 20 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 12, + "i": "13", + "w": 32, + "x": 0, + "y": 32 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0% - 17%": "rgb(255,255,204)", + "17% - 34%": "rgb(255,230,146)", + "34% - 50%": "rgb(254,191,90)", + "50% - 67%": "rgb(253,141,60)", + "67% - 84%": "rgb(244,61,37)", + "84% - 100%": "rgb(202,8,35)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 12, + "i": "15", + "w": 16, + "x": 32, + "y": 32 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Anti-Virus": "#EF843C", + "Content Security": "#7EB26D", + "Firewall": "#E24D42", + "Integrated Security": "#962D82", + "Network-based IDS/IPS": "#1F78C1", + "Operating System": "#1F78C1", + "VPN": "#EAB839" + } + } + }, + "gridData": { + "h": 12, + "i": "16", + "w": 16, + "x": 32, + "y": 20 + }, + "panelIndex": "16", + "panelRefName": "panel_16", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "17", + "w": 48, + "x": 0, + "y": 52 + }, + "panelIndex": "17", + "panelRefName": "panel_17", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 16, + "i": "18", + "w": 24, + "x": 0, + "y": 76 + }, + "panelIndex": "18", + "panelRefName": "panel_18", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "19", + "w": 24, + "x": 24, + "y": 76 + }, + "panelIndex": "19", + "panelRefName": "panel_19", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "20", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "20", + "panelRefName": "panel_20", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "21", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "21", + "panelRefName": "panel_21", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "c6a1fd07-de0f-444b-8814-902cbf2d019a", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "c1643919-b9de-4588-826f-93710a159e2b", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Destination Locations by Events — ArcSight [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "destination.geo.location", + "id": "5183bb72-a077-4cf0-8aba-561a15b012cf", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Destination Locations by Events — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 24, + "i": "49de47fb-1382-4009-89d2-b96a4161e12d", + "w": 24, + "x": 0, + "y": 92 + }, + "panelIndex": "49de47fb-1382-4009-89d2-b96a4161e12d", + "type": "map", + "version": "8.3.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "c2329af2-2183-45cb-9f40-d0f2e984c5b3", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "1fc250c2-4990-401e-b709-61e1f4824005", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Source Locations by Events — ArcSight [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "source.geo.location", + "id": "e1eda4fd-94b9-4c31-9615-70334517a966", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Source Locations by Events — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 24, + "i": "9d097034-9ebb-4f53-ad39-e42e625b541c", + "w": 24, + "x": 24, + "y": 92 + }, + "panelIndex": "9d097034-9ebb-4f53-ad39-e42e625b541c", + "type": "map", + "version": "8.3.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] ArcSight — Network Overview Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-346ac50b-0900-464d-aad6-007756a4e215", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-fe42b0d8-1fd3-4d8a-a4ff-4ca9915d2644", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-18e25223-652f-4768-9317-f611f8880313", + "name": "2:panel_2", + "type": "visualization" + }, + { + "id": "cef-1f1e8ad1-8232-4e9d-895b-096169da6e5a", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-29370aab-8b50-4553-b57e-f0c719cf00f5", + "name": "6:panel_6", + "type": "visualization" + }, + { + "id": "cef-05c3090d-ca0e-4e4a-8793-dee6fdf6134d", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-0e817af9-4cfe-42bf-823b-9fcc5e31fa79", + "name": "9:panel_9", + "type": "visualization" + }, + { + "id": "cef-0e4a06ad-6227-46df-8387-6d3987880709", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-43c3366e-6840-4956-8c82-dbe9b4eb9f16", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-d24ad888-a5cf-4b17-ac08-ae13d4cea23e", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "cef-3064f3fe-01de-4e6c-9e90-68961338c2ed", + "name": "16:panel_16", + "type": "visualization" + }, + { + "id": "cef-765d9f2a-7f2e-4765-94f3-77d9359ef50f", + "name": "17:panel_17", + "type": "visualization" + }, + { + "id": "cef-f1da0e00-c6a0-4e86-b9df-403f81818f62", + "name": "18:panel_18", + "type": "visualization" + }, + { + "id": "cef-15982ea6-25d7-4435-891e-a04fdfba6bf4", + "name": "19:panel_19", + "type": "visualization" + }, + { + "id": "cef-d618f6ab-e55c-42e9-b71e-f65e45630795", + "name": "20:panel_20", + "type": "visualization" + }, + { + "id": "cef-8ed5df68-1651-4e52-8c89-997109ddee63", + "name": "21:panel_21", + "type": "visualization" + }, + { + "id": "filebeat-*", + "name": "49de47fb-1382-4009-89d2-b96a4161e12d:layer_1_source_index_pattern", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "9d097034-9ebb-4f53-ad39-e42e625b541c:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-6d77b0a1-7b83-4c32-bbc5-cb1f6a4ee634.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-6d77b0a1-7b83-4c32-bbc5-cb1f6a4ee634.json new file mode 100644 index 00000000000..2e15ccf93ac --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-6d77b0a1-7b83-4c32-bbc5-cb1f6a4ee634.json @@ -0,0 +1,338 @@ +{ + "attributes": { + "description": "Suspicious network activity overview via ArcSight", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Destination Addresses": "#E0752D", + "Destination Ports": "#E24D42" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 12, + "i": "1", + "w": 32, + "x": 0, + "y": 28 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 12, + "i": "2", + "w": 16, + "x": 0, + "y": 40 + }, + "panelIndex": "2", + "panelRefName": "panel_2", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 12, + "i": "3", + "w": 16, + "x": 16, + "y": 40 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 20 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "/Attempt": "#0A50A1", + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "gridData": { + "h": 12, + "i": "9", + "w": 16, + "x": 32, + "y": 28 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "11", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "12", + "w": 24, + "x": 0, + "y": 52 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "13", + "w": 24, + "x": 24, + "y": 52 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "14", + "w": 16, + "x": 32, + "y": 40 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "15", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "16", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "16", + "panelRefName": "panel_16", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 50": "rgb(255,255,204)", + "100 - 200": "rgb(253,141,60)", + "200 - 300": "rgb(227,27,28)", + "300 - 400": "rgb(128,0,38)", + "50 - 100": "rgb(254,217,118)" + } + } + }, + "gridData": { + "h": 8, + "i": "17", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "17", + "panelRefName": "panel_17", + "type": "visualization", + "version": "7.3.0" + } + ], + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] ArcSight — Network Suspicious Activity Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-6d77b0a1-7b83-4c32-bbc5-cb1f6a4ee634", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-89236ffe-9484-472f-8e70-a58db4e2feb3", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-c59a4724-de59-4bdf-ba90-da482c6db8ad", + "name": "2:panel_2", + "type": "visualization" + }, + { + "id": "cef-9b0b7c11-1eaf-466c-8dab-4e3299d7091f", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-98511d76-8c58-4c0b-8723-6b495a0fd56e", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-eb86beed-6c68-4872-bb3e-e1df70f2c34a", + "name": "9:panel_9", + "type": "visualization" + }, + { + "id": "cef-9d5b10e7-e2be-4913-8f55-27bb72a0f787", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-17c61017-c26c-446c-9672-d98f71470d3a", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-463a7bb6-93ed-45f3-9489-c984b7b77401", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-23cb0c37-a1f7-4806-b3a2-580bd4bff3bd", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "cef-a1c8e760-dcca-4ae8-87d5-00461a179a5a", + "name": "16:panel_16", + "type": "visualization" + }, + { + "id": "cef-d618f6ab-e55c-42e9-b71e-f65e45630795", + "name": "17:panel_17", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-87e7074a-0764-4cdd-a39d-ec1be32c292f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-87e7074a-0764-4cdd-a39d-ec1be32c292f.json new file mode 100644 index 00000000000..b34f9777875 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-87e7074a-0764-4cdd-a39d-ec1be32c292f.json @@ -0,0 +1,433 @@ +{ + "attributes": { + "description": "Operating system activity from endpoints", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Count": "#64B0C8", + "Destination User Names": "#E24D42", + "Event Types": "#EF843C" + }, + "legendOpen": true + } + }, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 28 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "4", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "4", + "panelRefName": "panel_4", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 55k": "rgb(255,255,204)", + "110k - 165k": "rgb(254,225,135)", + "165k - 220k": "rgb(254,201,101)", + "220k - 275k": "rgb(254,171,73)", + "275k - 330k": "rgb(253,141,60)", + "330k - 385k": "rgb(252,91,46)", + "385k - 440k": "rgb(237,47,34)", + "440k - 495k": "rgb(212,16,32)", + "495k - 550k": "rgb(176,0,38)", + "55k - 110k": "rgb(255,241,170)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 12, + "i": "5", + "w": 24, + "x": 24, + "y": 28 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "7", + "w": 48, + "x": 0, + "y": 20 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "failure": "#E24D42", + "success": "#7EB26D", + "unknown": "#447EBC" + } + } + }, + "gridData": { + "h": 12, + "i": "8", + "w": 24, + "x": 24, + "y": 52 + }, + "panelIndex": "8", + "panelRefName": "panel_8", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 24, + "i": "9", + "w": 24, + "x": 0, + "y": 40 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "10", + "w": 24, + "x": 24, + "y": 40 + }, + "panelIndex": "10", + "panelRefName": "panel_10", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "11", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "12", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "Destination Users": "#E24D42", + "Event Count": "#64B0C8" + } + } + }, + "gridData": { + "h": 8, + "i": "13", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 20, + "i": "14", + "w": 16, + "x": 32, + "y": 64 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 24, + "i": "15", + "w": 16, + "x": 32, + "y": 84 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "16", + "w": 32, + "x": 0, + "y": 80 + }, + "panelIndex": "16", + "panelRefName": "panel_16", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "17", + "w": 32, + "x": 0, + "y": 100 + }, + "panelIndex": "17", + "panelRefName": "panel_17", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "18", + "w": 32, + "x": 0, + "y": 64 + }, + "panelIndex": "18", + "panelRefName": "panel_18", + "type": "visualization", + "version": "7.3.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "19", + "w": 32, + "x": 0, + "y": 92 + }, + "panelIndex": "19", + "panelRefName": "panel_19", + "type": "visualization", + "version": "7.3.0" + } + ], + "refreshInterval": { + "display": "Off", + "pause": false, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] Endpoint Activity Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-87e7074a-0764-4cdd-a39d-ec1be32c292f", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-20e3b246-944d-4a56-a36a-b07ba099a703", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-1ad0e81c-6182-47f5-94ca-16efd6b9654c", + "name": "4:panel_4", + "type": "visualization" + }, + { + "id": "cef-ea681829-8689-4f48-8930-76e6a78ac8ad", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-8d5ffb86-ef60-464e-82d9-d6f6de11cdd7", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-0fca030a-46e7-484e-a8b4-b39089f801d5", + "name": "8:panel_8", + "type": "visualization" + }, + { + "id": "cef-2c2fc6f9-0f61-44c4-ae7e-2d2cbc92fa44", + "name": "9:panel_9", + "type": "visualization" + }, + { + "id": "cef-74fb53d5-cba8-4bd4-9384-d831e706e0b4", + "name": "10:panel_10", + "type": "visualization" + }, + { + "id": "cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-78e69375-2f26-437f-a9b8-cb8d4bc26641", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-66484ff8-494f-4f55-8cc7-5e9ca57edf4c", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-08d9575c-7f11-4848-925f-c12e23e55987", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-a1aec4bd-be82-46bf-a35b-23cebee7b218", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "cef-6653dab8-40d4-47c0-8d4b-c14cc4cf3a67", + "name": "16:panel_16", + "type": "visualization" + }, + { + "id": "cef-e98f402f-d0ee-412a-995f-f3c455cac1ec", + "name": "17:panel_17", + "type": "visualization" + }, + { + "id": "cef-8922e448-709a-4040-b522-dc520a7cf711", + "name": "18:panel_18", + "type": "visualization" + }, + { + "id": "cef-e6d29418-b100-44f6-a0bb-059050e973ae", + "name": "19:panel_19", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNDksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-8d7c3098-7b2c-4d90-865b-75da6bfb29b6.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-8d7c3098-7b2c-4d90-865b-75da6bfb29b6.json new file mode 100644 index 00000000000..2f257fe093c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-8d7c3098-7b2c-4d90-865b-75da6bfb29b6.json @@ -0,0 +1,800 @@ +{ + "attributes": { + "description": "Network data overview", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "1", + "w": 48, + "x": 0, + "y": 32 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "2", + "w": 48, + "x": 0, + "y": 56 + }, + "panelIndex": "2", + "panelRefName": "panel_2", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "6", + "w": 48, + "x": 0, + "y": 48 + }, + "panelIndex": "6", + "panelRefName": "panel_6", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 8, + "i": "7", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "failure": "#BF1B00", + "success": "#629E51", + "unknown": "#0A50A1" + } + } + }, + "gridData": { + "h": 12, + "i": "11", + "w": 16, + "x": 16, + "y": 20 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 12, + "i": "13", + "w": 16, + "x": 0, + "y": 20 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0% - 17%": "rgb(255,255,204)", + "17% - 34%": "rgb(255,230,146)", + "34% - 50%": "rgb(254,191,90)", + "50% - 67%": "rgb(253,141,60)", + "67% - 84%": "rgb(244,61,37)", + "84% - 100%": "rgb(202,8,35)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 12, + "i": "15", + "w": 16, + "x": 32, + "y": 20 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "17", + "w": 48, + "x": 0, + "y": 40 + }, + "panelIndex": "17", + "panelRefName": "panel_17", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 16, + "i": "18", + "w": 24, + "x": 0, + "y": 64 + }, + "panelIndex": "18", + "panelRefName": "panel_18", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "19", + "w": 24, + "x": 24, + "y": 64 + }, + "panelIndex": "19", + "panelRefName": "panel_19", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "20", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "20", + "panelRefName": "panel_20", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "21", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "21", + "panelRefName": "panel_21", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "c6a1fd07-de0f-444b-8814-902cbf2d019a", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "c1643919-b9de-4588-826f-93710a159e2b", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Destination Locations by Events [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "destination.geo.location", + "id": "5183bb72-a077-4cf0-8aba-561a15b012cf", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Destination Locations by Events [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 24, + "i": "49de47fb-1382-4009-89d2-b96a4161e12d", + "w": 24, + "x": 0, + "y": 80 + }, + "panelIndex": "49de47fb-1382-4009-89d2-b96a4161e12d", + "type": "map", + "version": "8.3.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "c2329af2-2183-45cb-9f40-d0f2e984c5b3", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "1fc250c2-4990-401e-b709-61e1f4824005", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Source Locations by Events [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "source.geo.location", + "id": "e1eda4fd-94b9-4c31-9615-70334517a966", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Source Locations by Events [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 24, + "i": "9d097034-9ebb-4f53-ad39-e42e625b541c", + "w": 24, + "x": 24, + "y": 80 + }, + "panelIndex": "9d097034-9ebb-4f53-ad39-e42e625b541c", + "type": "map", + "version": "8.3.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] Network Overview Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8d7c3098-7b2c-4d90-865b-75da6bfb29b6", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-238866ef-33cc-4382-a30f-96f0c9a9a12b", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-e5126c06-0bbd-40d8-922f-3ece07b4eadf", + "name": "2:panel_2", + "type": "visualization" + }, + { + "id": "cef-e3ca7b8d-7c49-451a-a51e-6696ca9b223d", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-532bf41f-1a7e-4bb8-8704-5a5d6b6d92ea", + "name": "6:panel_6", + "type": "visualization" + }, + { + "id": "cef-d96eec3e-0f71-4949-81bb-216a5e9a6d08", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-42cad22a-93da-4747-9bd9-b2c0843bd7c8", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-82b8ee84-62bb-47b2-a865-a80caf4cb13a", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-645beb0d-5bee-4c0e-9172-56f67f877309", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "cef-beb62b4c-dbe4-4f72-8a70-5a102a9ae139", + "name": "17:panel_17", + "type": "visualization" + }, + { + "id": "cef-d2f2ec63-b44f-4b7c-8a48-9d457d974192", + "name": "18:panel_18", + "type": "visualization" + }, + { + "id": "cef-0f6894ae-882d-4b74-85e6-8381541f3bcd", + "name": "19:panel_19", + "type": "visualization" + }, + { + "id": "cef-4e4c17aa-f137-4cfd-80dd-7519f72bb7af", + "name": "20:panel_20", + "type": "visualization" + }, + { + "id": "cef-bc37e59b-a9b4-4de1-8f24-2706b65b888a", + "name": "21:panel_21", + "type": "visualization" + }, + { + "id": "filebeat-*", + "name": "49de47fb-1382-4009-89d2-b96a4161e12d:layer_1_source_index_pattern", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "9d097034-9ebb-4f53-ad39-e42e625b541c:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-b16c980c-21ad-47ca-8103-7ce949ddc87a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-b16c980c-21ad-47ca-8103-7ce949ddc87a.json new file mode 100644 index 00000000000..a7a473803bc --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-b16c980c-21ad-47ca-8103-7ce949ddc87a.json @@ -0,0 +1,754 @@ +{ + "attributes": { + "description": "Overview of Microsoft DNS activity via ArcSight", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "1", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "3", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 18k": "rgb(247,251,255)", + "108k - 126k": "rgb(74,152,201)", + "126k - 144k": "rgb(46,126,188)", + "144k - 162k": "rgb(23,100,171)", + "162k - 180k": "rgb(8,74,145)", + "18k - 36k": "rgb(227,238,249)", + "36k - 54k": "rgb(208,225,242)", + "54k - 72k": "rgb(182,212,233)", + "72k - 90k": "rgb(148,196,223)", + "90k - 108k": "rgb(107,174,214)" + }, + "legendOpen": false + } + }, + "gridData": { + "h": 16, + "i": "5", + "w": 24, + "x": 0, + "y": 32 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "6", + "w": 48, + "x": 0, + "y": 48 + }, + "panelIndex": "6", + "panelRefName": "panel_6", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 16, + "i": "7", + "w": 24, + "x": 24, + "y": 32 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "9", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 16, + "i": "11", + "w": 24, + "x": 24, + "y": 56 + }, + "panelIndex": "11", + "panelRefName": "panel_11", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "12", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 16, + "i": "13", + "w": 24, + "x": 0, + "y": 56 + }, + "panelIndex": "13", + "panelRefName": "panel_13", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "14", + "w": 24, + "x": 0, + "y": 20 + }, + "panelIndex": "14", + "panelRefName": "panel_14", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "15", + "w": 24, + "x": 24, + "y": 20 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "56b3b288-a0f1-416d-9d40-96a37c8484fd", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "d50cbece-4556-4421-bb06-fb015bfe7baa", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Sources by Events — ArcSight [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "source.geo.location", + "id": "555cbeac-b098-4946-9498-6b700e745e8a", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Sources by Events — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 12, + "i": "3cf2118b-5231-49f5-b685-0ff0e1f52c32", + "w": 24, + "x": 0, + "y": 72 + }, + "panelIndex": "3cf2118b-5231-49f5-b685-0ff0e1f52c32", + "type": "map", + "version": "8.3.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "5231e15c-d374-46ca-9553-3308d723ded3", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "8cdaae20-5dcc-4930-b105-802fc344fcb6", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Destinations by Events — ArcSight [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "destination.geo.location", + "id": "88700fdc-3a96-46b8-b51f-3839111eb6ec", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Destinations by Events — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -90 + }, + "mapCenter": { + "lat": 16.40767, + "lon": 0, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 12, + "i": "07f92eca-2078-4aa6-8373-d27ca33595d6", + "w": 24, + "x": 24, + "y": 72 + }, + "panelIndex": "07f92eca-2078-4aa6-8373-d27ca33595d6", + "type": "map", + "version": "8.3.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] ArcSight — Microsoft DNS Overview", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-b16c980c-21ad-47ca-8103-7ce949ddc87a", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-bd8f3914-cf95-4451-bd26-482130922a4f", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-509321f8-3864-4435-8ca7-7e9b0fd382e0", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-2f43e7a2-abaa-40d3-b2a1-1961954f141d", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-47afc6a2-7281-4f95-a1e8-75db6b7b62b4", + "name": "6:panel_6", + "type": "visualization" + }, + { + "id": "cef-8d4f3630-04f0-4b40-8d56-5b989b471370", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-5db7c0d4-ca64-4e8e-aae3-15742cafd85e", + "name": "9:panel_9", + "type": "visualization" + }, + { + "id": "cef-30c6520b-371f-4add-b70f-526397e5670e", + "name": "11:panel_11", + "type": "visualization" + }, + { + "id": "cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-d134c1d0-2493-41ea-9ba7-c449a8ec0a6c", + "name": "13:panel_13", + "type": "visualization" + }, + { + "id": "cef-b8f5c70f-5dfa-4cdc-8bbd-7cefcee4c902", + "name": "14:panel_14", + "type": "visualization" + }, + { + "id": "cef-22cf043b-2633-4041-807a-b1ba9dcb1de5", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "filebeat-*", + "name": "3cf2118b-5231-49f5-b685-0ff0e1f52c32:layer_1_source_index_pattern", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "07f92eca-2078-4aa6-8373-d27ca33595d6:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-c5ea76fb-5d0b-462d-922b-978ccd1f0f5c.json b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-c5ea76fb-5d0b-462d-922b-978ccd1f0f5c.json new file mode 100644 index 00000000000..80bbc6c9134 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/dashboard/cef-c5ea76fb-5d0b-462d-922b-978ccd1f0f5c.json @@ -0,0 +1,515 @@ +{ + "attributes": { + "description": "Summary of endpoint event data", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "event.dataset:cef.log" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "1", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "1", + "panelRefName": "panel_1", + "type": "visualization", + "version": "8.2.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "failure": "#BF1B00", + "success": "#629E51", + "unknown": "#0A50A1" + } + } + }, + "gridData": { + "h": 12, + "i": "2", + "w": 24, + "x": 24, + "y": 20 + }, + "panelIndex": "2", + "panelRefName": "panel_2", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "colors": { + "failure": "#BF1B00", + "success": "#629E51", + "unknown": "#0A50A1" + } + } + }, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 20 + }, + "panelIndex": "3", + "panelRefName": "panel_3", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "5", + "w": 48, + "x": 0, + "y": 12 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 12, + "i": "6", + "w": 24, + "x": 24, + "y": 32 + }, + "panelIndex": "6", + "panelRefName": "panel_6", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "gridData": { + "h": 8, + "i": "7", + "w": 40, + "x": 0, + "y": 4 + }, + "panelIndex": "7", + "panelRefName": "panel_7", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "columns": [ + "cef.extensions.categoryDeviceGroup", + "cef.extensions.categoryTechnique", + "event.outcome", + "event.category", + "event.type", + "cef.extensions.categoryObject", + "event.action", + "cef.extensions.categoryDeviceType" + ], + "enhancements": {}, + "sort": [ + "@timestamp", + "desc" + ] + }, + "gridData": { + "h": 20, + "i": "9", + "w": 48, + "x": 0, + "y": 72 + }, + "panelIndex": "9", + "panelRefName": "panel_9", + "type": "search", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 8, + "i": "10", + "w": 24, + "x": 24, + "y": 44 + }, + "panelIndex": "10", + "panelRefName": "panel_10", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "gridData": { + "h": 20, + "i": "12", + "w": 24, + "x": 0, + "y": 32 + }, + "panelIndex": "12", + "panelRefName": "panel_12", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 4, + "i": "15", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "15", + "panelRefName": "panel_15", + "type": "visualization", + "version": "8.0.0" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "layerListJSON": [ + { + "alpha": 1, + "id": "de084257-24da-4ea9-922e-a2d7565ebcd6", + "includeInFitToBounds": true, + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "lightModeDefault": "road_map_desaturated", + "type": "EMS_TMS" + }, + "style": { + "type": "TILE" + }, + "type": "EMS_VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "741ceaa6-5b51-4959-9935-c5961b12f539", + "includeInFitToBounds": true, + "joins": [], + "label": "Top Destination Locations by Event [Filebeat CEF]", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyForceRefresh": true, + "applyGlobalQuery": true, + "applyGlobalTime": true, + "geoField": "destination.geo.location", + "id": "ba850a09-c635-4855-b68b-de16dd200d6f", + "indexPatternRefName": "layer_1_source_index_pattern", + "metrics": [ + { + "type": "count" + } + ], + "requestType": "point", + "resolution": "MOST_FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "type": "ORDINAL" + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#3d3d3d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "GEOJSON_VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 16.40767, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": true + }, + "settings": { + "autoFitToDataBounds": false, + "backgroundColor": "#ffffff", + "browserLocation": { + "zoom": 2 + }, + "disableInteractive": false, + "disableTooltipControl": false, + "fixedLocation": { + "lat": 0, + "lon": 0, + "zoom": 2 + }, + "hideLayerControl": false, + "hideToolbarOverlay": false, + "hideViewControl": false, + "initialLocation": "LAST_SAVED_LOCATION", + "maxZoom": 24, + "minZoom": 0, + "showScaleControl": false, + "showSpatialFilters": true, + "showTimesliderToggleButton": true, + "spatialFiltersAlpa": 0.3, + "spatialFiltersFillColor": "#DA8B45", + "spatialFiltersLineColor": "#DA8B45" + }, + "timeFilters": { + "from": "now-24h", + "to": "now" + }, + "zoom": 1.78 + }, + "references": [], + "title": "Top Destination Locations by Event [Filebeat CEF]", + "uiStateJSON": { + "isLayerTOCOpen": true, + "openTOCDetails": [] + } + }, + "enhancements": {}, + "hiddenLayers": [], + "isLayerTOCOpen": true, + "mapBuffer": { + "maxLat": 66.51326, + "maxLon": 90, + "minLat": -66.51326, + "minLon": -180 + }, + "mapCenter": { + "lat": 20.86831, + "lon": -12.2843, + "zoom": 1.78 + }, + "openTOCDetails": [], + "type": "map" + }, + "gridData": { + "h": 20, + "i": "c9fd3ece-2bef-4cdc-9f83-ed689b35a17a", + "w": 48, + "x": 0, + "y": 52 + }, + "panelIndex": "c9fd3ece-2bef-4cdc-9f83-ed689b35a17a", + "type": "map", + "version": "8.3.0" + } + ], + "refreshInterval": { + "pause": true, + "value": 0 + }, + "timeFrom": "now-24h", + "timeRestore": true, + "timeTo": "now", + "title": "[Filebeat CEF] Endpoint Overview Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-c5ea76fb-5d0b-462d-922b-978ccd1f0f5c", + "migrationVersion": { + "dashboard": "8.3.0" + }, + "references": [ + { + "id": "cef-fbaeed43-bca4-4206-9a42-935140b2b48e", + "name": "1:panel_1", + "type": "visualization" + }, + { + "id": "cef-f1582760-9a38-44f8-bc1f-9832b967a034", + "name": "2:panel_2", + "type": "visualization" + }, + { + "id": "cef-7556bb16-ddfb-4aca-b351-662bbb699093", + "name": "3:panel_3", + "type": "visualization" + }, + { + "id": "cef-9cebd9ff-0094-40f3-bed7-45dea62a5279", + "name": "5:panel_5", + "type": "visualization" + }, + { + "id": "cef-1db6b68d-d52b-4822-884a-1d6bd396dabb", + "name": "6:panel_6", + "type": "visualization" + }, + { + "id": "cef-56ef4e25-7603-40cd-a538-dd722bfb36d2", + "name": "7:panel_7", + "type": "visualization" + }, + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "9:panel_9", + "type": "search" + }, + { + "id": "cef-51075bf3-ea47-4e9a-b7d1-fedcfcb6c981", + "name": "10:panel_10", + "type": "visualization" + }, + { + "id": "cef-e345045a-26d7-44bf-8829-0d48d78419b5", + "name": "12:panel_12", + "type": "visualization" + }, + { + "id": "cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927", + "name": "15:panel_15", + "type": "visualization" + }, + { + "id": "filebeat-*", + "name": "c9fd3ece-2bef-4cdc-9f83-ed689b35a17a:layer_1_source_index_pattern", + "type": "index-pattern" + } + ], + "type": "dashboard", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-12352cbd-ccbb-4ede-a98b-c4c82a358516.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-12352cbd-ccbb-4ede-a98b-c4c82a358516.json new file mode 100644 index 00000000000..4389d30c474 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-12352cbd-ccbb-4ede-a98b-c4c82a358516.json @@ -0,0 +1,57 @@ +{ + "attributes": { + "columns": [ + "cef.device.vendor", + "cef.device.product", + "message", + "cef.device.event_class_id", + "cef.extensions.deviceEventCategory", + "source.user.name", + "destination.user.name", + "destination.domain", + "event.action", + "event.outcome", + "cef.extensions.sourceNtDomain", + "cef.extensions.destinationNtDomain" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "event.dataset:\"cef.log\"" + } + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Endpoint - Events [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1.json new file mode 100644 index 00000000000..c728e0d1c5c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1.json @@ -0,0 +1,87 @@ +{ + "attributes": { + "columns": [ + "priority", + "message", + "source.ip", + "source.port", + "destination.ip", + "destination.port", + "network.application", + "message", + "cef.extensions.categoryBehavior", + "cef.extensions.categoryOutcome", + "cef.extensions.deviceAddress", + "cef.device.product", + "cef.device.vendor", + "cef.extensions.categoryDeviceGroup", + "cef.extensions.categoryDeviceType" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "query", + "negate": false, + "type": "custom", + "value": "{\"terms\":{\"cef.extensions.categoryDeviceGroup\":[\"/VPN\",\"/IDS/Network\",\"/Firewall\"]}}" + }, + "query": { + "terms": { + "cef.extensions.categoryDeviceGroup": [ + "/VPN", + "/IDS/Network", + "/Firewall" + ] + } + } + } + ], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "lucene", + "query": "" + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Network Events — ArcSight [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-2f6c8e48-8c6a-476d-82e8-47c723c39715.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-2f6c8e48-8c6a-476d-82e8-47c723c39715.json new file mode 100644 index 00000000000..29ce1a13213 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-2f6c8e48-8c6a-476d-82e8-47c723c39715.json @@ -0,0 +1,52 @@ +{ + "attributes": { + "columns": [ + "cef.extensions.categoryDeviceGroup", + "cef.extensions.categoryTechnique", + "cef.extensions.categoryOutcome", + "cef.extensions.categorySignificance", + "cef.extensions.categoryObject", + "cef.extensions.categoryBehavior", + "cef.extensions.categoryDeviceType" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Host\" OR cef.extensions.categoryDeviceGroup:\"/Application\"" + } + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Endpoint Event Explorer — ArcSight [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-69d6e511-7744-429a-9aa4-ceae2222db94.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-69d6e511-7744-429a-9aa4-ceae2222db94.json new file mode 100644 index 00000000000..fd9bbdbe4c4 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-69d6e511-7744-429a-9aa4-ceae2222db94.json @@ -0,0 +1,57 @@ +{ + "attributes": { + "columns": [ + "cef.device.vendor", + "cef.device.product", + "message", + "cef.device.event_class_id", + "cef.extensions.deviceEventCategory", + "source.user.name", + "destination.user.name", + "destination.domain", + "cef.extensions.categoryBehavior", + "cef.extensions.categoryOutcome", + "cef.extensions.sourceNtDomain", + "cef.extensions.destinationNtDomain" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\"" + } + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Endpoint - OS Events — ArcSight [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e.json new file mode 100644 index 00000000000..552c7150e2e --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e.json @@ -0,0 +1,68 @@ +{ + "attributes": { + "columns": [ + "cef.device.vendor", + "cef.device.product", + "event.action", + "event.outcome", + "destination.ip", + "destination.port", + "destination.domain", + "cef.device.event_class_id", + "cef.extensions.deviceCustomString1Label", + "cef.extensions.deviceCustomString1", + "cef.extensions.deviceCustomString2Label", + "cef.extensions.deviceCustomString2", + "cef.extension.deviceCustomString3Label", + "cef.extension.deviceCustomString3", + "cef.extension.deviceCustomString4Label", + "cef.extension.deviceCustomString4", + "cef.extensions.deviceEventCategory", + "event.severity", + "source.ip", + "source.port", + "network.transport", + "source.bytes", + "url.original" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "cef.device.product:\"DNS Trace Log\"" + } + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Microsoft DNS Events [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-721d1d17-9c3a-4002-9f23-d51a12604d41.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-721d1d17-9c3a-4002-9f23-d51a12604d41.json new file mode 100644 index 00000000000..dff4a938011 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-721d1d17-9c3a-4002-9f23-d51a12604d41.json @@ -0,0 +1,68 @@ +{ + "attributes": { + "columns": [ + "cef.device.vendor", + "cef.device.product", + "cef.extensions.categoryBehavior", + "cef.extensions.categoryOutcome", + "destination.ip", + "destination.port", + "destination.domain", + "cef.device.event_class_id", + "cef.extensions.deviceCustomString1Label", + "cef.extensions.deviceCustomString1", + "cef.extensions.deviceCustomString2Label", + "cef.extensions.deviceCustomString2", + "cef.extension.deviceCustomString3Label", + "cef.extension.deviceCustomString3", + "cef.extension.deviceCustomString4Label", + "cef.extension.deviceCustomString4", + "cef.extensions.deviceEventCategory", + "event.severity", + "source.ip", + "source.port", + "network.transport", + "source.bytes", + "url.original" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "cef.device.product:\"DNS Trace Log\"" + } + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Microsoft DNS Events — ArcSight [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-8eaeaa88-771f-4721-8f35-f94da4ea6009.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-8eaeaa88-771f-4721-8f35-f94da4ea6009.json new file mode 100644 index 00000000000..91b33856cc5 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-8eaeaa88-771f-4721-8f35-f94da4ea6009.json @@ -0,0 +1,53 @@ +{ + "attributes": { + "columns": [ + "cef.extensions.categoryDeviceGroup", + "cef.extensions.categoryTechnique", + "event.outcome", + "event.category", + "event.type", + "cef.extensions.categoryObject", + "event.action", + "cef.extensions.categoryDeviceType" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "event.dataset:\"cef.log\"" + } + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Endpoint Event Explorer [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-d687d631-96df-4c49-99ee-9e51c7053084.json b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-d687d631-96df-4c49-99ee-9e51c7053084.json new file mode 100644 index 00000000000..60283c41bb8 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/search/cef-d687d631-96df-4c49-99ee-9e51c7053084.json @@ -0,0 +1,83 @@ +{ + "attributes": { + "columns": [ + "priority", + "message", + "source.ip", + "source.port", + "destination.ip", + "destination.port", + "network.application", + "message", + "event.action", + "event.outcome", + "cef.extensions.deviceAddress", + "cef.device.product", + "cef.device.vendor", + "cef.extensions.categoryDeviceGroup", + "cef.extensions.categoryDeviceType" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "query", + "negate": false, + "type": "custom", + "value": "{\"term\":{\"event.category\":\"network\"}}" + }, + "query": { + "term": { + "event.category": "network" + } + } + } + ], + "highlightAll": true, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "lucene", + "query": "" + }, + "version": true + } + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "Network Events [Filebeat CEF]", + "version": 1 + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-01c35a60-419a-4703-ba46-ca6a8a6d1acf.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-01c35a60-419a-4703-ba46-ca6a8a6d1acf.json new file mode 100644 index 00000000000..3b2f87ca65b --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-01c35a60-419a-4703-ba46-ca6a8a6d1acf.json @@ -0,0 +1,152 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Outcomes Breakdown — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "/Attempt": "#3F2B5B", + "/Failure": "#BF1B00" + }, + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Time", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Time" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "area", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Outcomes Breakdown — ArcSight [Filebeat CEF]", + "type": "area" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-01c35a60-419a-4703-ba46-ca6a8a6d1acf", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-02b3c5cc-240f-4ca5-b49f-56e2f1cc83fc.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-02b3c5cc-240f-4ca5-b49f-56e2f1cc83fc.json new file mode 100644 index 00000000000..776011e1d65 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-02b3c5cc-240f-4ca5-b49f-56e2f1cc83fc.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Users — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Users — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-02b3c5cc-240f-4ca5-b49f-56e2f1cc83fc", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-05c3090d-ca0e-4e4a-8793-dee6fdf6134d.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-05c3090d-ca0e-4e4a-8793-dee6fdf6134d.json new file mode 100644 index 00000000000..d80036e3364 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-05c3090d-ca0e-4e4a-8793-dee6fdf6134d.json @@ -0,0 +1,135 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Device Metrics Overview — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Sources", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destinations", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "12", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Device Metrics Overview — ArcSight [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-05c3090d-ca0e-4e4a-8793-dee6fdf6134d", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-08d9575c-7f11-4848-925f-c12e23e55987.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-08d9575c-7f11-4848-925f-c12e23e55987.json new file mode 100644 index 00000000000..3b6375c32bf --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-08d9575c-7f11-4848-925f-c12e23e55987.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Sources by Destinations [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Host", + "field": "source.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination Host", + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "bottom", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 10 Sources by Destinations [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-08d9575c-7f11-4848-925f-c12e23e55987", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-08dd95b3-aa74-4271-9c53-30e106fd5ea6.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-08dd95b3-aa74-4271-9c53-30e106fd5ea6.json new file mode 100644 index 00000000000..ded33441ccf --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-08dd95b3-aa74-4271-9c53-30e106fd5ea6.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Users by Destination Users — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Users", + "field": "source.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "bottom", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 10 Source Users by Destination Users — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-08dd95b3-aa74-4271-9c53-30e106fd5ea6", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-09047d0c-900c-4949-9cf6-122154cc203f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-09047d0c-900c-4949-9cf6-122154cc203f.json new file mode 100644 index 00000000000..d0f568ea1ae --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-09047d0c-900c-4949-9cf6-122154cc203f.json @@ -0,0 +1,114 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Behaviors by Outcome — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 9,000": "rgb(255,255,204)", + "18,000 - 27,000": "rgb(254,225,135)", + "27,000 - 36,000": "rgb(254,201,101)", + "36,000 - 45,000": "rgb(254,171,73)", + "45,000 - 54,000": "rgb(253,141,60)", + "54,000 - 63,000": "rgb(252,91,46)", + "63,000 - 72,000": "rgb(237,47,34)", + "72,000 - 81,000": "rgb(212,16,32)", + "81,000 - 90,000": "rgb(176,0,38)", + "9,000 - 18,000": "rgb(255,241,170)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Type", + "field": "cef.extensions.categoryBehavior", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Yellow to Red", + "colorsNumber": 10, + "colorsRange": [], + "enableHover": true, + "invertColors": false, + "legendPosition": "right", + "legendSize": "auto", + "percentageMode": false, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "Top 10 Behaviors by Outcome — ArcSight [Filebeat CEF]", + "type": "heatmap" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-09047d0c-900c-4949-9cf6-122154cc203f", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-09816af7-d246-4880-b4c8-1ace7ea35b3b.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-09816af7-d246-4880-b4c8-1ace7ea35b3b.json new file mode 100644 index 00000000000..043f032375f --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-09816af7-d246-4880-b4c8-1ace7ea35b3b.json @@ -0,0 +1,135 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Device Metrics Overview [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Sources", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destinations", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "12", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Device Metrics Overview [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-09816af7-d246-4880-b4c8-1ace7ea35b3b", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNTksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0af23787-8d02-4028-9f41-f32398fefbd0.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0af23787-8d02-4028-9f41-f32398fefbd0.json new file mode 100644 index 00000000000..0d02398c45a --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0af23787-8d02-4028-9f41-f32398fefbd0.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Port — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Port — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-0af23787-8d02-4028-9f41-f32398fefbd0", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0b7d6680-54e6-4afd-9748-a686ad4d72f0.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0b7d6680-54e6-4afd-9748-a686ad4d72f0.json new file mode 100644 index 00000000000..3ed198dd955 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0b7d6680-54e6-4afd-9748-a686ad4d72f0.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destinations — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Hosts", + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destinations — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-0b7d6680-54e6-4afd-9748-a686ad4d72f0", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0e4a06ad-6227-46df-8387-6d3987880709.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0e4a06ad-6227-46df-8387-6d3987880709.json new file mode 100644 index 00000000000..18981a2e9d0 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0e4a06ad-6227-46df-8387-6d3987880709.json @@ -0,0 +1,153 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Destination Ports by Outcome — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Protocols", + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "rotate": 75, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Protocols" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "percentage", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Destination Ports by Outcome — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-0e4a06ad-6227-46df-8387-6d3987880709", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0e817af9-4cfe-42bf-823b-9fcc5e31fa79.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0e817af9-4cfe-42bf-823b-9fcc5e31fa79.json new file mode 100644 index 00000000000..9f0ffd598f6 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0e817af9-4cfe-42bf-823b-9fcc5e31fa79.json @@ -0,0 +1,153 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Outcome by Device Type — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Firewall Types", + "field": "cef.extensions.categoryDeviceType", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 3 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "rotate": 75, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Firewall Types" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "orderBucketsBySum": true, + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "percentage", + "type": "square root" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Outcome by Device Type — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-0e817af9-4cfe-42bf-823b-9fcc5e31fa79", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0f6894ae-882d-4b74-85e6-8381541f3bcd.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0f6894ae-882d-4b74-85e6-8381541f3bcd.json new file mode 100644 index 00000000000..aa281dbb420 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0f6894ae-882d-4b74-85e6-8381541f3bcd.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 20 Source Countries [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 26, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "square root" + }, + "title": "Top 20 Source Countries [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-0f6894ae-882d-4b74-85e6-8381541f3bcd", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0fca030a-46e7-484e-a8b4-b39089f801d5.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0fca030a-46e7-484e-a8b4-b39089f801d5.json new file mode 100644 index 00000000000..61a400e09c8 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-0fca030a-46e7-484e-a8b4-b39089f801d5.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 20 Behaviors by Outcome [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Behavior", + "field": "event.action", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "event.outcome", + "order": "desc", + "orderBy": "1", + "size": 3 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 20 Behaviors by Outcome [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-0fca030a-46e7-484e-a8b4-b39089f801d5", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1013a3cb-ca15-459e-b1a6-45f110958b5c.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1013a3cb-ca15-459e-b1a6-45f110958b5c.json new file mode 100644 index 00000000000..3bc6f319a99 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1013a3cb-ca15-459e-b1a6-45f110958b5c.json @@ -0,0 +1,123 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "DNS Metrics Overview [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Threads", + "field": "cef.extensions.deviceCustomString1" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "OpCodes", + "field": "cef.extensions.deviceCustomString2" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Activity Types", + "field": "cef.device.event_class_id" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "32", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "type": "gauge" + }, + "title": "DNS Metrics Overview [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1013a3cb-ca15-459e-b1a6-45f110958b5c", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-119bcab5-9bb0-4644-8dfa-15ff8848c0a2.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-119bcab5-9bb0-4644-8dfa-15ff8848c0a2.json new file mode 100644 index 00000000000..730d2bbd5ab --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-119bcab5-9bb0-4644-8dfa-15ff8848c0a2.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Event Types [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "cef.device.event_class_id", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 50, + "minFontSize": 12, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "square root" + }, + "title": "Top 10 Event Types [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-119bcab5-9bb0-4644-8dfa-15ff8848c0a2", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-15982ea6-25d7-4435-891e-a04fdfba6bf4.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-15982ea6-25d7-4435-891e-a04fdfba6bf4.json new file mode 100644 index 00000000000..f94e02bf735 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-15982ea6-25d7-4435-891e-a04fdfba6bf4.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 20 Source Countries — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 26, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "square root" + }, + "title": "Top 20 Source Countries — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-15982ea6-25d7-4435-891e-a04fdfba6bf4", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-17c61017-c26c-446c-9672-d98f71470d3a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-17c61017-c26c-446c-9672-d98f71470d3a.json new file mode 100644 index 00000000000..82de12fa3d0 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-17c61017-c26c-446c-9672-d98f71470d3a.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Addresses — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Source Addresses — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-17c61017-c26c-446c-9672-d98f71470d3a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNjksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-18a1d5b4-8f9e-43b6-a6e7-d9f24db85136.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-18a1d5b4-8f9e-43b6-a6e7-d9f24db85136.json new file mode 100644 index 00000000000..20ea5ca80eb --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-18a1d5b4-8f9e-43b6-a6e7-d9f24db85136.json @@ -0,0 +1,117 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 15 Event Types by Events — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Types", + "field": "cef.extensions.categoryBehavior", + "order": "desc", + "orderBy": "1", + "size": 15 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Users", + "field": "source.user.name" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Source Hosts", + "field": "source.domain" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destination Hosts", + "field": "destination.domain" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + } + ], + "listeners": {}, + "params": { + "perPage": 15, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 15 Event Types by Events — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-18a1d5b4-8f9e-43b6-a6e7-d9f24db85136", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-18e25223-652f-4768-9317-f611f8880313.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-18e25223-652f-4768-9317-f611f8880313.json new file mode 100644 index 00000000000..1aad2237dd4 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-18e25223-652f-4768-9317-f611f8880313.json @@ -0,0 +1,117 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Bandwidth Utilization — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "kuery", + "query": "" + }, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "d27f09dc-b07e-493f-a223-a85033ad6548", + "label": "Inbound", + "line_width": 1, + "metrics": [ + { + "field": "source.bytes", + "id": "9ce9ec3a-2f11-4935-91b2-531494d2a619", + "type": "sum" + } + ], + "override_index_pattern": 1, + "point_size": 1, + "seperate_axis": 0, + "series_drop_last_bucket": 1, + "series_index_pattern": "filebeat-*", + "series_time_field": "@timestamp", + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "terms_field": "observer.hostname", + "terms_order_by": "_count" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "b1ef2c75-5916-469d-8790-5b213367a5a0", + "label": "Outbound", + "line_width": 1, + "metrics": [ + { + "field": "destination.bytes", + "id": "11b1852f-9b62-4e96-8128-522e6c5bf16d", + "type": "sum" + }, + { + "id": "2a6b00bf-1658-4d02-b4e2-61ad6e4c3a9b", + "script": "params.outbound \u003e 0 ? params.outbound * -1 : 0", + "type": "calculation", + "variables": [ + { + "field": "11b1852f-9b62-4e96-8128-522e6c5bf16d", + "id": "c57067f2-2927-41d8-97f4-9f47b3b3bcae", + "name": "outbound" + } + ] + } + ], + "override_index_pattern": 1, + "point_size": 1, + "seperate_axis": 0, + "series_drop_last_bucket": 1, + "series_index_pattern": "filebeat-*", + "series_time_field": "@timestamp", + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "steps": 0 + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Bandwidth Utilization — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-18e25223-652f-4768-9317-f611f8880313", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1a9f1963-aa1c-418f-8c15-74dbbcd87758.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1a9f1963-aa1c-418f-8c15-74dbbcd87758.json new file mode 100644 index 00000000000..4d29b1a8f4a --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1a9f1963-aa1c-418f-8c15-74dbbcd87758.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Ports [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Ports [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1a9f1963-aa1c-418f-8c15-74dbbcd87758", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1ad0e81c-6182-47f5-94ca-16efd6b9654c.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1ad0e81c-6182-47f5-94ca-16efd6b9654c.json new file mode 100644 index 00000000000..08cd47f9234 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1ad0e81c-6182-47f5-94ca-16efd6b9654c.json @@ -0,0 +1,125 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Endpoint OS Metrics Overview [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total Events" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Event Types", + "field": "event.action" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Event Outcomes", + "field": "event.outcome" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "20", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Endpoint OS Metrics Overview [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1ad0e81c-6182-47f5-94ca-16efd6b9654c", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1db6b68d-d52b-4822-884a-1d6bd396dabb.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1db6b68d-d52b-4822-884a-1d6bd396dabb.json new file mode 100644 index 00000000000..bda2cf4cd91 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1db6b68d-d52b-4822-884a-1d6bd396dabb.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Port [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Port [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1db6b68d-d52b-4822-884a-1d6bd396dabb", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1e9875ef-0cfd-40d1-b261-e02838274a7c.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1e9875ef-0cfd-40d1-b261-e02838274a7c.json new file mode 100644 index 00000000000..6aa1fb753bb --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1e9875ef-0cfd-40d1-b261-e02838274a7c.json @@ -0,0 +1,118 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events Types by Severity [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "id": "db54ebce-9dd2-4a1e-b476-b3ddb9a9024e", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": "0", + "formatter": "number", + "id": "81da76ca-1112-4d91-82f4-c66cd3156a84", + "label": "Cumulative Bytes", + "line_width": "3", + "metrics": [ + { + "field": "source.bytes", + "id": "521d560c-321a-4410-9eb3-2b2bf3f4efee", + "type": "count" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "(event.severity:\"2\" OR event.severity:\"3\" OR event.severity:\"5\" OR event.severity:\"16\" OR cef.extension.deviceCustomString4:\"SERVFAIL\" OR cef.extension.deviceCustomString4:\"NXDOMAIN\" OR cef.extension.deviceCustomString4:\"REFUSED\" OR cef.extension.deviceCustomString4:\"BADVERS\" OR cef.extension.deviceCustomString4:\"BADSIG\")" + }, + "id": "3f31a7e4-acf3-4f2d-8b7d-e30522325b2a", + "label": "HIGH" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "(event.severity:\"1\" OR event.severity:\"4\" OR event.severity:\"6\" OR event.severity:\"7\" OR event.severity:\"8\" OR event.severity:\"9\" OR event.severity:\"10\" OR event.severity:\"17\" OR event.severity:\"18\" OR event.severity:\"19\" OR event.severity:\"20\" OR event.severity:\"21\" OR event.severity:\"22\" OR cef.extension.deviceCustomString4:\"Error\" OR cef.extension.deviceCustomString4:\"ERROR\" OR cef.extension.deviceCustomString4:\"Warning\" OR cef.extension.deviceCustomString4:\"WARNING\" OR cef.extension.deviceCustomString4:\"FORMERR\" OR cef.extension.deviceCustomString4:\"NOTIMP\" OR cef.extension.deviceCustomString4:\"YXDOMAIN\" OR cef.extension.deviceCustomString4:\"YXRRSET\" OR cef.extension.deviceCustomString4:\"NXRRSET\" OR cef.extension.deviceCustomString4:\"NOTAUTH\" OR cef.extension.deviceCustomString4:\"NOTZONE\" OR cef.extension.deviceCustomString4:\"BADKEY\" OR cef.extension.deviceCustomString4:\"BADTIME\" OR cef.extension.deviceCustomString4:\"BADMODE\" OR cef.extension.deviceCustomString4:\"BADNAME\" OR cef.extension.deviceCustomString4:\"BADALG\" OR cef.extension.deviceCustomString4:\"BADTRUNC\")" + }, + "id": "7949d31b-8aae-433a-b7cf-6939a8728cc9", + "label": "MEDIUM" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "(NOT (event.severity:\"2\" OR event.severity:\"3\" OR event.severity:\"5\" OR event.severity:\"16\" OR cef.extension.deviceCustomString4:\"SERVFAIL\" OR cef.extension.deviceCustomString4:\"NXDOMAIN\" OR cef.extension.deviceCustomString4:\"REFUSED\" OR cef.extension.deviceCustomString4:\"BADVERS\" OR cef.extension.deviceCustomString4:\"BADSIG\" OR event.severity:\"1\" OR event.severity:\"4\" OR event.severity:\"6\" OR event.severity:\"7\" OR event.severity:\"8\" OR event.severity:\"9\" OR event.severity:\"10\" OR event.severity:\"17\" OR event.severity:\"18\" OR event.severity:\"19\" OR event.severity:\"20\" OR event.severity:\"21\" OR event.severity:\"22\" OR cef.extension.deviceCustomString4:\"Error\" OR cef.extension.deviceCustomString4:\"ERROR\" OR cef.extension.deviceCustomString4:\"Warning\" OR cef.extension.deviceCustomString4:\"WARNING\" OR cef.extension.deviceCustomString4:\"FORMERR\" OR cef.extension.deviceCustomString4:\"NOTIMP\" OR cef.extension.deviceCustomString4:\"YXDOMAIN\" OR cef.extension.deviceCustomString4:\"YXRRSET\" OR cef.extension.deviceCustomString4:\"NXRRSET\" OR cef.extension.deviceCustomString4:\"NOTAUTH\" OR cef.extension.deviceCustomString4:\"NOTZONE\" OR cef.extension.deviceCustomString4:\"BADKEY\" OR cef.extension.deviceCustomString4:\"BADTIME\" OR cef.extension.deviceCustomString4:\"BADMODE\" OR cef.extension.deviceCustomString4:\"BADNAME\" OR cef.extension.deviceCustomString4:\"BADALG\" OR cef.extension.deviceCustomString4:\"BADTRUNC\"))" + }, + "id": "d2627211-5f9e-4c65-8a47-1cd6f085939d", + "label": "LOW" + } + ], + "split_mode": "filters", + "stacked": "none" + }, + { + "axis_position": "right", + "chart_type": "bar", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "a5fda184-fdd6-4221-ab59-492eab162f0a", + "label": "Count by Event Type", + "line_width": 1, + "metrics": [ + { + "id": "e147ba1c-b13a-496f-9841-b99ddee81c5a", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "cef.device.event_class_id", + "terms_size": "20" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events Types by Severity [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1e9875ef-0cfd-40d1-b261-e02838274a7c", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1f1e8ad1-8232-4e9d-895b-096169da6e5a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1f1e8ad1-8232-4e9d-895b-096169da6e5a.json new file mode 100644 index 00000000000..66917368a1f --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1f1e8ad1-8232-4e9d-895b-096169da6e5a.json @@ -0,0 +1,168 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Source — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "2fddda5e-d6fc-4581-bbb7-574e1017ae8f" + } + ], + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "gauge_color_rules": [ + { + "id": "3ed9a6b9-fd2e-4e0d-bd83-7ad467b3c8a4" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": "0", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "e5a48d9d-7834-4da7-8d78-7d4528136b9b", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "sigma": "", + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "id": "0c929603-fc92-4ebc-a963-fe2795417d89", + "label": "Firewall Events" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/IDS/Network\"" + }, + "id": "7798827b-87ab-436b-9e62-9fe36143eb9b", + "label": "Intrusion Detection Events" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "id": "490f7ad7-8218-45f9-85a9-a4dd9ed7da13", + "label": "VPN" + } + ], + "split_mode": "filters", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostname", + "terms_order_by": null + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(0,156,224,1)", + "fill": "0.5", + "formatter": "number", + "id": "29d6131a-5143-4a64-b597-9538692f0269", + "label": "Moving Average by Device Hosts", + "line_width": 1, + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "gamma": 0.3, + "id": "87e21aaa-12eb-4213-bb37-41cb19219240", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "observer.hostname", + "terms_size": "10" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Source — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1f1e8ad1-8232-4e9d-895b-096169da6e5a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1f4fa9d2-128f-4fca-bb20-f3fca2be6961.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1f4fa9d2-128f-4fca-bb20-f3fca2be6961.json new file mode 100644 index 00000000000..d7f13dd447b --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-1f4fa9d2-128f-4fca-bb20-f3fca2be6961.json @@ -0,0 +1,140 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Source Addresses [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "a0bf5a1d-8ebf-49d4-a347-738a6ce20562" + } + ], + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\" " + }, + "gauge_color_rules": [ + { + "id": "42f84a0a-ee13-4ca8-b61d-3de482ae4ab0" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(211,49,21,1)", + "fill": "0", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\" " + }, + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "117fde19-e227-4fcb-8019-e82e6677c340", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "sigma": "", + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostmessage", + "terms_order_by": null, + "value_template": "{{value}}" + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(104,188,0,1)", + "fill": "0.5", + "formatter": "number", + "id": "3ffe652e-43c2-4a1d-ad8a-f7ab10f09f2b", + "label": "Top Source Addresses", + "line_width": "0", + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "gamma": 0.3, + "id": "b753ad38-c3ed-4463-8f6d-176f4d477897", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "source.ip", + "terms_size": "10" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Source Addresses [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-1f4fa9d2-128f-4fca-bb20-f3fca2be6961", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-20e3b246-944d-4a56-a36a-b07ba099a703.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-20e3b246-944d-4a56-a36a-b07ba099a703.json new file mode 100644 index 00000000000..3268be39d24 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-20e3b246-944d-4a56-a36a-b07ba099a703.json @@ -0,0 +1,202 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Source Users by Event Type and Destination Users [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Users", + "field": "source.user.name", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Types", + "field": "event.action" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination User Names", + "field": "destination.user.name" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Source Users" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Event Types" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "stacked", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + }, + { + "data": { + "id": "4", + "label": "Destination User Names" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "stacked", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "" + }, + "type": "value" + } + ] + }, + "title": "Source Users by Event Type and Destination Users [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-20e3b246-944d-4a56-a36a-b07ba099a703", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-22cf043b-2633-4041-807a-b1ba9dcb1de5.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-22cf043b-2633-4041-807a-b1ba9dcb1de5.json new file mode 100644 index 00000000000..f0c76f5bc69 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-22cf043b-2633-4041-807a-b1ba9dcb1de5.json @@ -0,0 +1,108 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Size — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "id": "6e634117-6b30-411c-b74c-75510befe42f", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "filter": { + "language": "lucene", + "query": "deviceDirection:\"0\"" + }, + "formatter": "bytes", + "id": "28b1fb5b-0f16-4519-b901-4dd2dcc39915", + "label": "Inbound Bytes", + "line_width": "2", + "metrics": [ + { + "field": "source.bytes", + "id": "f613f33f-6459-4e46-a3a0-c36c48c46b2e", + "type": "sum" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "filter", + "stacked": "none" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": 0.5, + "filter": { + "language": "lucene", + "query": "deviceDirection:\"1\"" + }, + "formatter": "bytes", + "id": "5a5c2529-4990-4006-b039-c94069ff6b7e", + "label": "Outbound Bytes", + "line_width": "2", + "metrics": [ + { + "field": "source.bytes", + "id": "b69501e7-56d5-4c38-81d1-34d778c81e11", + "type": "sum" + }, + { + "id": "0aaab374-5845-44ab-94f5-ac4fab25c287", + "script": "params.outbound_bytes \u003e= 0 ? params.outbound_bytes * -1 : 0", + "type": "calculation", + "variables": [ + { + "field": "b69501e7-56d5-4c38-81d1-34d778c81e11", + "id": "23b8c41c-0e98-4ace-8bca-3593e46cd955", + "name": "outbound_bytes" + } + ] + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "filter", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Size — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-22cf043b-2633-4041-807a-b1ba9dcb1de5", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzNzksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-238866ef-33cc-4382-a30f-96f0c9a9a12b.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-238866ef-33cc-4382-a30f-96f0c9a9a12b.json new file mode 100644 index 00000000000..488ac5a94f8 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-238866ef-33cc-4382-a30f-96f0c9a9a12b.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Application Protocols [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "network.application", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 26, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "square root" + }, + "title": "Top 10 Application Protocols [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-238866ef-33cc-4382-a30f-96f0c9a9a12b", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-23cb0c37-a1f7-4806-b3a2-580bd4bff3bd.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-23cb0c37-a1f7-4806-b3a2-580bd4bff3bd.json new file mode 100644 index 00000000000..abfb227c956 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-23cb0c37-a1f7-4806-b3a2-580bd4bff3bd.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Ports — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Ports — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-23cb0c37-a1f7-4806-b3a2-580bd4bff3bd", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-29370aab-8b50-4553-b57e-f0c719cf00f5.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-29370aab-8b50-4553-b57e-f0c719cf00f5.json new file mode 100644 index 00000000000..81b6bb3ea61 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-29370aab-8b50-4553-b57e-f0c719cf00f5.json @@ -0,0 +1,169 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Outcome — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "2fddda5e-d6fc-4581-bbb7-574e1017ae8f" + } + ], + "bar_color_rules": [ + { + "bar_color": null, + "id": "23db5bf6-f787-474e-86ab-76362432e984", + "value": 0 + } + ], + "drilldown_url": "", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "gauge_color_rules": [ + { + "id": "3ed9a6b9-fd2e-4e0d-bd83-7ad467b3c8a4" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(211,49,21,1)", + "fill": "0", + "filter": { + "language": "lucene", + "query": "(cef.extensions.categoryDeviceGroup:\"/Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\") AND _exists_:cef.extensions.categoryOutcome" + }, + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "c43af7e6-3f06-48a4-a7c3-7ba8bd6214f9", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "id": "4c7aac7d-2749-41b6-8136-40dc8636a7e7", + "label": "Firewall" + } + ], + "split_mode": "filter", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostname", + "terms_order_by": null + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(104,188,0,1)", + "fill": "1", + "formatter": "number", + "id": "29d6131a-5143-4a64-b597-9538692f0269", + "label": "Moving Average by Event Outcome", + "line_width": 1, + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(104,188,0,0.35)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Success\"" + }, + "id": "cb1ae397-13a0-4b6f-a848-bcdc96870f05", + "label": "Success" + }, + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Failure\"" + }, + "id": "ef021c15-1b95-4334-bc3c-e2950e9b0f6f", + "label": "Failure" + }, + { + "color": "rgba(0,156,224,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Attempt\"" + }, + "id": "2ff1e859-b178-4824-a0f2-69a115932b98", + "label": "Attempt" + } + ], + "split_mode": "filters", + "stacked": "stacked", + "terms_field": "cef.extensions.categoryOutcome", + "terms_size": "3" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Outcome — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-29370aab-8b50-4553-b57e-f0c719cf00f5", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-2c2fc6f9-0f61-44c4-ae7e-2d2cbc92fa44.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-2c2fc6f9-0f61-44c4-ae7e-2d2cbc92fa44.json new file mode 100644 index 00000000000..d44f508bbaf --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-2c2fc6f9-0f61-44c4-ae7e-2d2cbc92fa44.json @@ -0,0 +1,117 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 15 Event Types by Events [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Types", + "field": "event.action", + "order": "desc", + "orderBy": "1", + "size": 15 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Users", + "field": "source.user.name" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Source Hosts", + "field": "source.domain" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destination Hosts", + "field": "destination.domain" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + } + ], + "listeners": {}, + "params": { + "perPage": 15, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 15 Event Types by Events [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-2c2fc6f9-0f61-44c4-ae7e-2d2cbc92fa44", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-2f43e7a2-abaa-40d3-b2a1-1961954f141d.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-2f43e7a2-abaa-40d3-b2a1-1961954f141d.json new file mode 100644 index 00000000000..82714d22fce --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-2f43e7a2-abaa-40d3-b2a1-1961954f141d.json @@ -0,0 +1,132 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top Destinations by Traffic Size — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 18k": "rgb(247,251,255)", + "108k - 126k": "rgb(74,152,201)", + "126k - 144k": "rgb(46,126,188)", + "144k - 162k": "rgb(23,100,171)", + "162k - 180k": "rgb(8,74,145)", + "18k - 36k": "rgb(227,238,249)", + "36k - 54k": "rgb(208,225,242)", + "54k - 72k": "rgb(182,212,233)", + "72k - 90k": "rgb(148,196,223)", + "90k - 108k": "rgb(107,174,214)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "2", + "params": { + "filters": [ + { + "input": { + "language": "lucene", + "query": "deviceDirection:\"0\"" + }, + "label": "Inbound" + }, + { + "input": { + "language": "lucene", + "query": "deviceDirection:\"1\"" + }, + "label": "Outbound" + } + ] + }, + "schema": "segment", + "type": "filters" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Blues", + "colorsNumber": 10, + "colorsRange": [ + { + "from": 0, + "to": null + } + ], + "enableHover": true, + "invertColors": false, + "legendPosition": "top", + "legendSize": "auto", + "percentageMode": false, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "Top Destinations by Traffic Size — ArcSight [Filebeat CEF]", + "type": "heatmap" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-2f43e7a2-abaa-40d3-b2a1-1961954f141d", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-3064f3fe-01de-4e6c-9e90-68961338c2ed.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-3064f3fe-01de-4e6c-9e90-68961338c2ed.json new file mode 100644 index 00000000000..3ce184fd45a --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-3064f3fe-01de-4e6c-9e90-68961338c2ed.json @@ -0,0 +1,68 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Device Type Breakdown — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Firewall Types", + "field": "cef.extensions.categoryDeviceType", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": false, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Device Type Breakdown — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-3064f3fe-01de-4e6c-9e90-68961338c2ed", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-30c6520b-371f-4add-b70f-526397e5670e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-30c6520b-371f-4add-b70f-526397e5670e.json new file mode 100644 index 00000000000..ec2f69035ff --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-30c6520b-371f-4add-b70f-526397e5670e.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destinations by Size — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destinations", + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Sources", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Count" + }, + "schema": "metric", + "type": "count" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Destinations by Size — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-30c6520b-371f-4add-b70f-526397e5670e", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-3ab5a5fa-3775-49d3-892b-4362531971cd.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-3ab5a5fa-3775-49d3-892b-4362531971cd.json new file mode 100644 index 00000000000..799386e014f --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-3ab5a5fa-3775-49d3-892b-4362531971cd.json @@ -0,0 +1,104 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Endpoint - OS Average EPS — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "bar_color_rules": [ + { + "id": "ce9549a0-3af0-4070-b169-4b6d145d4c39" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\"" + }, + "gauge_color_rules": [ + { + "id": "03a2fd72-fc9c-4582-9133-20af36217180" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "94161c6c-4f48-4beb-9d78-f79f29c02a34", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "b4373ffd-9660-4206-afd6-d4867ac7dbdf", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "type": "count" + }, + { + "field": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "id": "89f8286e-4aec-4cb4-83ad-b139692edf3d", + "type": "cumulative_sum" + }, + { + "field": "89f8286e-4aec-4cb4-83ad-b139692edf3d", + "id": "1df39e5f-3e98-4ed7-ab08-47f3ca2ee915", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "1df39e5f-3e98-4ed7-ab08-47f3ca2ee915", + "gamma": 0.3, + "id": "f46a6e6e-444f-4c7e-b5eb-e1a59568f2eb", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "offset_time": "1m", + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "Endpoint - OS Average EPS — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-3ab5a5fa-3775-49d3-892b-4362531971cd", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-42cad22a-93da-4747-9bd9-b2c0843bd7c8.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-42cad22a-93da-4747-9bd9-b2c0843bd7c8.json new file mode 100644 index 00000000000..3b6e70bab63 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-42cad22a-93da-4747-9bd9-b2c0843bd7c8.json @@ -0,0 +1,153 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Destination Ports by Outcome [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "failure": "#BF1B00", + "success": "#629E51" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Protocols", + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "event.outcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "rotate": 75, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Protocols" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "percentage", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Destination Ports by Outcome [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-42cad22a-93da-4747-9bd9-b2c0843bd7c8", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-42d96ddb-1e83-49fc-91e6-fb189e0b2661.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-42d96ddb-1e83-49fc-91e6-fb189e0b2661.json new file mode 100644 index 00000000000..3a07bb62dc6 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-42d96ddb-1e83-49fc-91e6-fb189e0b2661.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Device Types by Vendor — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "exclude": "Network-based IDS/IPS", + "field": "cef.extensions.categoryDeviceType", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "exclude": "", + "field": "cef.device.vendor", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": false, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Device Types by Vendor — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-42d96ddb-1e83-49fc-91e6-fb189e0b2661", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzODksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-43c3366e-6840-4956-8c82-dbe9b4eb9f16.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-43c3366e-6840-4956-8c82-dbe9b4eb9f16.json new file mode 100644 index 00000000000..dbdf724adfe --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-43c3366e-6840-4956-8c82-dbe9b4eb9f16.json @@ -0,0 +1,120 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Devices by Bandwidth — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Device", + "field": "observer.hostname", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Source(s)", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Destination(s)", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Bandwidth (Incoming)", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Bandwidth (Outgoing)", + "field": "destination.bytes" + }, + "schema": "metric", + "type": "sum" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Devices by Bandwidth — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-43c3366e-6840-4956-8c82-dbe9b4eb9f16", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4410b326-f66e-4dc6-88f7-8ea234087581.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4410b326-f66e-4dc6-88f7-8ea234087581.json new file mode 100644 index 00000000000..f9f3599bed1 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4410b326-f66e-4dc6-88f7-8ea234087581.json @@ -0,0 +1,135 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Endpoint Metrics Overview — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Port", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "12", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Endpoint Metrics Overview — ArcSight [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-4410b326-f66e-4dc6-88f7-8ea234087581", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-463a7bb6-93ed-45f3-9489-c984b7b77401.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-463a7bb6-93ed-45f3-9489-c984b7b77401.json new file mode 100644 index 00000000000..6caef0a51c5 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-463a7bb6-93ed-45f3-9489-c984b7b77401.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Addresses — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Addresses — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-463a7bb6-93ed-45f3-9489-c984b7b77401", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-46f5dcfe-2d61-4022-9cbb-2b044135fc15.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-46f5dcfe-2d61-4022-9cbb-2b044135fc15.json new file mode 100644 index 00000000000..bda4809eefa --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-46f5dcfe-2d61-4022-9cbb-2b044135fc15.json @@ -0,0 +1,107 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Outcomes by User Names — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "/Informational": "#7EB26D", + "/Informational/Warning": "#EF843C", + "/Success": "#64B0C8", + "Anti-Virus": "#B7DBAB", + "Host-based IDS/IPS": "#629E51", + "Log Consolidator": "#E0F9D7", + "Operating System": "#3F6833", + "Recon": "#BF1B00", + "Security Mangement": "#CFFAFF" + }, + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "exclude": "Network-based IDS/IPS", + "field": "cef.extensions.categoryDeviceType", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "6", + "params": { + "field": "destination.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": false, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Outcomes by User Names — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-46f5dcfe-2d61-4022-9cbb-2b044135fc15", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-47afc6a2-7281-4f95-a1e8-75db6b7b62b4.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-47afc6a2-7281-4f95-a1e8-75db6b7b62b4.json new file mode 100644 index 00000000000..65ff2b1c1c2 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-47afc6a2-7281-4f95-a1e8-75db6b7b62b4.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Event Types — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "cef.device.event_class_id", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 50, + "minFontSize": 12, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "square root" + }, + "title": "Top 10 Event Types — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-47afc6a2-7281-4f95-a1e8-75db6b7b62b4", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4e4c17aa-f137-4cfd-80dd-7519f72bb7af.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4e4c17aa-f137-4cfd-80dd-7519f72bb7af.json new file mode 100644 index 00000000000..4639d3d3614 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4e4c17aa-f137-4cfd-80dd-7519f72bb7af.json @@ -0,0 +1,111 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Network - Event Throughput [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color_rules": [ + { + "id": "3eadd451-5033-423f-88e3-814cc5e50b50" + } + ], + "bar_color_rules": [ + { + "id": "8d4596c5-49ad-429b-af54-5451b1c2e8d4" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\" " + }, + "gauge_color_rules": [ + { + "gauge": null, + "id": "4d957654-cc7e-4ef3-8b29-61c0aeadd51a", + "value": 0 + } + ], + "gauge_inner_width": 10, + "gauge_max": "", + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "73968651-c41e-473e-a153-a025f49d1a1b", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "90d7621e-3265-4fe8-8882-8df9605ea659", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "type": "count" + }, + { + "field": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "id": "ca3a65d0-9f3d-42a9-9f4e-16f9e24cba19", + "type": "cumulative_sum" + }, + { + "field": "ca3a65d0-9f3d-42a9-9f4e-16f9e24cba19", + "id": "6db67bc1-7fff-47e7-a931-f797b1f76732", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "6db67bc1-7fff-47e7-a931-f797b1f76732", + "gamma": 0.3, + "id": "92bc1447-2b30-498c-ae8a-c67904fc82b2", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "Network - Event Throughput [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-4e4c17aa-f137-4cfd-80dd-7519f72bb7af", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4fe11728-f553-403f-8dda-952e28acb607.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4fe11728-f553-403f-8dda-952e28acb607.json new file mode 100644 index 00000000000..ec89694b84f --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-4fe11728-f553-403f-8dda-952e28acb607.json @@ -0,0 +1,108 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Countries by Event — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total Events" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 35 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Source Countries by Event — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-4fe11728-f553-403f-8dda-952e28acb607", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-509321f8-3864-4435-8ca7-7e9b0fd382e0.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-509321f8-3864-4435-8ca7-7e9b0fd382e0.json new file mode 100644 index 00000000000..a9b3756c954 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-509321f8-3864-4435-8ca7-7e9b0fd382e0.json @@ -0,0 +1,123 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "DNS Metrics Overview — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Threads", + "field": "cef.extensions.deviceCustomString1" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "OpCodes", + "field": "cef.extensions.deviceCustomString2" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Activity Types", + "field": "cef.device.event_class_id" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "32", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "type": "gauge" + }, + "title": "DNS Metrics Overview — ArcSight [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-509321f8-3864-4435-8ca7-7e9b0fd382e0", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-51075bf3-ea47-4e9a-b7d1-fedcfcb6c981.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-51075bf3-ea47-4e9a-b7d1-fedcfcb6c981.json new file mode 100644 index 00000000000..4676b97f1c8 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-51075bf3-ea47-4e9a-b7d1-fedcfcb6c981.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Source Countries [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 5 Source Countries [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-51075bf3-ea47-4e9a-b7d1-fedcfcb6c981", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-532bf41f-1a7e-4bb8-8704-5a5d6b6d92ea.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-532bf41f-1a7e-4bb8-8704-5a5d6b6d92ea.json new file mode 100644 index 00000000000..27089e6e6b2 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-532bf41f-1a7e-4bb8-8704-5a5d6b6d92ea.json @@ -0,0 +1,169 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Outcome [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "2fddda5e-d6fc-4581-bbb7-574e1017ae8f" + } + ], + "bar_color_rules": [ + { + "bar_color": null, + "id": "23db5bf6-f787-474e-86ab-76362432e984", + "value": 0 + } + ], + "drilldown_url": "", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "gauge_color_rules": [ + { + "id": "3ed9a6b9-fd2e-4e0d-bd83-7ad467b3c8a4" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(211,49,21,1)", + "fill": "0", + "filter": { + "language": "lucene", + "query": "(cef.extensions.categoryDeviceGroup:\"/Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\") AND _exists_:cef.extensions.categoryOutcome" + }, + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "c43af7e6-3f06-48a4-a7c3-7ba8bd6214f9", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "id": "4c7aac7d-2749-41b6-8136-40dc8636a7e7", + "label": "Firewall" + } + ], + "split_mode": "filter", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostname", + "terms_order_by": null + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(104,188,0,1)", + "fill": "1", + "formatter": "number", + "id": "29d6131a-5143-4a64-b597-9538692f0269", + "label": "Moving Average by Event Outcome", + "line_width": 1, + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(104,188,0,0.35)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Success\"" + }, + "id": "cb1ae397-13a0-4b6f-a848-bcdc96870f05", + "label": "Success" + }, + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Failure\"" + }, + "id": "ef021c15-1b95-4334-bc3c-e2950e9b0f6f", + "label": "Failure" + }, + { + "color": "rgba(0,156,224,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Attempt\"" + }, + "id": "2ff1e859-b178-4824-a0f2-69a115932b98", + "label": "Attempt" + } + ], + "split_mode": "filters", + "stacked": "stacked", + "terms_field": "event.outcome", + "terms_size": "3" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Outcome [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-532bf41f-1a7e-4bb8-8704-5a5d6b6d92ea", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzIzOTksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-53b315dc-f7f1-444b-8556-8f1cd5dae16a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-53b315dc-f7f1-444b-8556-8f1cd5dae16a.json new file mode 100644 index 00000000000..785cf4db758 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-53b315dc-f7f1-444b-8556-8f1cd5dae16a.json @@ -0,0 +1,115 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Severity [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "bar_color_rules": [ + { + "id": "0ca18a89-9c81-4bee-835a-85e6103aec37" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "hide_last_value_indicator": true, + "id": "c39a76e5-f613-41a9-8335-c442747791e0", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "0.0[0]a", + "id": "da3b92b4-2c24-473b-9102-fb5a343a96d9", + "label": "Event by Severities", + "line_width": 1, + "metrics": [ + { + "id": "0d189776-3f7c-4a92-95b1-73c379a341fc", + "type": "count" + }, + { + "field": "0d189776-3f7c-4a92-95b1-73c379a341fc", + "id": "1b1c931c-a09b-4980-af81-6f9c3db56401", + "sigma": "", + "type": "sum_bucket" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(104,204,202,1)", + "filter": { + "language": "lucene", + "query": "severity:\"Low\" OR severity:\"0\"" + }, + "id": "ebe970ac-5cc9-4c4a-af60-82affafc667c", + "label": "LOW" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "severity:\"Medium\"" + }, + "id": "0c4ff16a-b53d-4ce4-af76-d6b74d8788db", + "label": "MEDIUM" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "severity:\"High\"" + }, + "id": "e142c55b-6ee5-416a-8bd3-d10398044864", + "label": "HIGH" + }, + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "severity:\"Very-High\"" + }, + "id": "4b05b562-c419-4214-b814-d4c242251521", + "label": "VERY HIGH" + } + ], + "split_mode": "filters", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "top_n", + "use_kibana_indexes": false + }, + "title": "Events by Severity [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-53b315dc-f7f1-444b-8556-8f1cd5dae16a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-56ef4e25-7603-40cd-a538-dd722bfb36d2.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-56ef4e25-7603-40cd-a538-dd722bfb36d2.json new file mode 100644 index 00000000000..eaf5e9849f4 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-56ef4e25-7603-40cd-a538-dd722bfb36d2.json @@ -0,0 +1,135 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Endpoint Metrics Overview [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Port", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "12", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Endpoint Metrics Overview [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-56ef4e25-7603-40cd-a538-dd722bfb36d2", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-5db7c0d4-ca64-4e8e-aae3-15742cafd85e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-5db7c0d4-ca64-4e8e-aae3-15742cafd85e.json new file mode 100644 index 00000000000..2d01f2331a0 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-5db7c0d4-ca64-4e8e-aae3-15742cafd85e.json @@ -0,0 +1,118 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events Types by Severity — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "id": "db54ebce-9dd2-4a1e-b476-b3ddb9a9024e", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": "0", + "formatter": "number", + "id": "81da76ca-1112-4d91-82f4-c66cd3156a84", + "label": "Cumulative Bytes", + "line_width": "3", + "metrics": [ + { + "field": "source.bytes", + "id": "521d560c-321a-4410-9eb3-2b2bf3f4efee", + "type": "count" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "(event.severity:\"2\" OR event.severity:\"3\" OR event.severity:\"5\" OR event.severity:\"16\" OR cef.extension.deviceCustomString4:\"SERVFAIL\" OR cef.extension.deviceCustomString4:\"NXDOMAIN\" OR cef.extension.deviceCustomString4:\"REFUSED\" OR cef.extension.deviceCustomString4:\"BADVERS\" OR cef.extension.deviceCustomString4:\"BADSIG\")" + }, + "id": "3f31a7e4-acf3-4f2d-8b7d-e30522325b2a", + "label": "HIGH" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "(event.severity:\"1\" OR event.severity:\"4\" OR event.severity:\"6\" OR event.severity:\"7\" OR event.severity:\"8\" OR event.severity:\"9\" OR event.severity:\"10\" OR event.severity:\"17\" OR event.severity:\"18\" OR event.severity:\"19\" OR event.severity:\"20\" OR event.severity:\"21\" OR event.severity:\"22\" OR cef.extension.deviceCustomString4:\"Error\" OR cef.extension.deviceCustomString4:\"ERROR\" OR cef.extension.deviceCustomString4:\"Warning\" OR cef.extension.deviceCustomString4:\"WARNING\" OR cef.extension.deviceCustomString4:\"FORMERR\" OR cef.extension.deviceCustomString4:\"NOTIMP\" OR cef.extension.deviceCustomString4:\"YXDOMAIN\" OR cef.extension.deviceCustomString4:\"YXRRSET\" OR cef.extension.deviceCustomString4:\"NXRRSET\" OR cef.extension.deviceCustomString4:\"NOTAUTH\" OR cef.extension.deviceCustomString4:\"NOTZONE\" OR cef.extension.deviceCustomString4:\"BADKEY\" OR cef.extension.deviceCustomString4:\"BADTIME\" OR cef.extension.deviceCustomString4:\"BADMODE\" OR cef.extension.deviceCustomString4:\"BADNAME\" OR cef.extension.deviceCustomString4:\"BADALG\" OR cef.extension.deviceCustomString4:\"BADTRUNC\")" + }, + "id": "7949d31b-8aae-433a-b7cf-6939a8728cc9", + "label": "MEDIUM" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "(NOT (event.severity:\"2\" OR event.severity:\"3\" OR event.severity:\"5\" OR event.severity:\"16\" OR cef.extension.deviceCustomString4:\"SERVFAIL\" OR cef.extension.deviceCustomString4:\"NXDOMAIN\" OR cef.extension.deviceCustomString4:\"REFUSED\" OR cef.extension.deviceCustomString4:\"BADVERS\" OR cef.extension.deviceCustomString4:\"BADSIG\" OR event.severity:\"1\" OR event.severity:\"4\" OR event.severity:\"6\" OR event.severity:\"7\" OR event.severity:\"8\" OR event.severity:\"9\" OR event.severity:\"10\" OR event.severity:\"17\" OR event.severity:\"18\" OR event.severity:\"19\" OR event.severity:\"20\" OR event.severity:\"21\" OR event.severity:\"22\" OR cef.extension.deviceCustomString4:\"Error\" OR cef.extension.deviceCustomString4:\"ERROR\" OR cef.extension.deviceCustomString4:\"Warning\" OR cef.extension.deviceCustomString4:\"WARNING\" OR cef.extension.deviceCustomString4:\"FORMERR\" OR cef.extension.deviceCustomString4:\"NOTIMP\" OR cef.extension.deviceCustomString4:\"YXDOMAIN\" OR cef.extension.deviceCustomString4:\"YXRRSET\" OR cef.extension.deviceCustomString4:\"NXRRSET\" OR cef.extension.deviceCustomString4:\"NOTAUTH\" OR cef.extension.deviceCustomString4:\"NOTZONE\" OR cef.extension.deviceCustomString4:\"BADKEY\" OR cef.extension.deviceCustomString4:\"BADTIME\" OR cef.extension.deviceCustomString4:\"BADMODE\" OR cef.extension.deviceCustomString4:\"BADNAME\" OR cef.extension.deviceCustomString4:\"BADALG\" OR cef.extension.deviceCustomString4:\"BADTRUNC\"))" + }, + "id": "d2627211-5f9e-4c65-8a47-1cd6f085939d", + "label": "LOW" + } + ], + "split_mode": "filters", + "stacked": "none" + }, + { + "axis_position": "right", + "chart_type": "bar", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "a5fda184-fdd6-4221-ab59-492eab162f0a", + "label": "Count by Event Type", + "line_width": 1, + "metrics": [ + { + "id": "e147ba1c-b13a-496f-9841-b99ddee81c5a", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "cef.device.event_class_id", + "terms_size": "20" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events Types by Severity — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-5db7c0d4-ca64-4e8e-aae3-15742cafd85e", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927.json new file mode 100644 index 00000000000..d30a682ae5c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "query_string": { + "query": "*" + } + } + } + }, + "title": " Dashboard Navigation [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "markdown": "[Network Overview](#/dashboard/cef-8d7c3098-7b2c-4d90-865b-75da6bfb29b6) | [Network Suspicious Activity](#/dashboard/cef-2745d8ff-e261-428b-a406-15ee77c8a3dd) | [Endpoint Overview](#dashboard/cef-c5ea76fb-5d0b-462d-922b-978ccd1f0f5c) | [Endpoint Activity](#/dashboard/cef-87e7074a-0764-4cdd-a39d-ec1be32c292f) | [Microsoft DNS Overview](#/dashboard/cef-244684fd-7be2-49fb-a817-e51cec267bf1)" + }, + "title": " Dashboard Navigation [Filebeat CEF]", + "type": "markdown" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-5e4a73ac-198d-45b3-b1d8-41cb8a242927", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-645beb0d-5bee-4c0e-9172-56f67f877309.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-645beb0d-5bee-4c0e-9172-56f67f877309.json new file mode 100644 index 00000000000..43ee7a88730 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-645beb0d-5bee-4c0e-9172-56f67f877309.json @@ -0,0 +1,110 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Devices by Outcome [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0% - 17%": "rgb(255,255,204)", + "17% - 34%": "rgb(255,230,146)", + "34% - 50%": "rgb(254,191,90)", + "50% - 67%": "rgb(253,141,60)", + "67% - 84%": "rgb(244,61,37)", + "84% - 100%": "rgb(202,8,35)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Device Host Names", + "field": "observer.hostname", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "event.outcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Yellow to Red", + "colorsNumber": 6, + "colorsRange": [], + "enableHover": true, + "invertColors": false, + "legendPosition": "right", + "legendSize": "auto", + "percentageMode": true, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "Top 10 Devices by Outcome [Filebeat CEF]", + "type": "heatmap" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-645beb0d-5bee-4c0e-9172-56f67f877309", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-66484ff8-494f-4f55-8cc7-5e9ca57edf4c.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-66484ff8-494f-4f55-8cc7-5e9ca57edf4c.json new file mode 100644 index 00000000000..4966f90d4ac --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-66484ff8-494f-4f55-8cc7-5e9ca57edf4c.json @@ -0,0 +1,204 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Events by Source and Destination Users [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Timestamp", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Users", + "field": "source.user.name" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Timestamp" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Event Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Source Users" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 3, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + }, + { + "data": { + "id": "4", + "label": "Destination Users" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 3, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Event Count" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "" + }, + "type": "value" + } + ] + }, + "title": "Events by Source and Destination Users [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-66484ff8-494f-4f55-8cc7-5e9ca57edf4c", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6653dab8-40d4-47c0-8d4b-c14cc4cf3a67.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6653dab8-40d4-47c0-8d4b-c14cc4cf3a67.json new file mode 100644 index 00000000000..66f37c3f2e7 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6653dab8-40d4-47c0-8d4b-c14cc4cf3a67.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destinations [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Hosts", + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destinations [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-6653dab8-40d4-47c0-8d4b-c14cc4cf3a67", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-66c201f1-2ffb-4eea-b408-a59315998364.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-66c201f1-2ffb-4eea-b408-a59315998364.json new file mode 100644 index 00000000000..a53a63ee18e --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-66c201f1-2ffb-4eea-b408-a59315998364.json @@ -0,0 +1,103 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Endpoint Average EPS — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "bar_color_rules": [ + { + "id": "85a1c642-9781-430d-b84b-b28cb2a42fb4" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Host\" OR cef.extensions.categoryDeviceGroup:\"/Application\"" + }, + "gauge_color_rules": [ + { + "id": "03a2fd72-fc9c-4582-9133-20af36217180" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "b7a85957-123e-4e25-9e8e-ff7992c9b2b9", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "b4373ffd-9660-4206-afd6-d4867ac7dbdf", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "type": "count" + }, + { + "field": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "id": "7c5c44cc-17bd-4206-a100-b8996cd3d11a", + "type": "cumulative_sum" + }, + { + "field": "7c5c44cc-17bd-4206-a100-b8996cd3d11a", + "id": "215c5225-5368-40e6-8fcd-2b0026babba0", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "215c5225-5368-40e6-8fcd-2b0026babba0", + "gamma": 0.3, + "id": "f4dfe09a-e397-4287-ab99-3206516cded3", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "Endpoint Average EPS — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-66c201f1-2ffb-4eea-b408-a59315998364", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6a5a6bfa-8f61-418e-8fc4-0c5d8ddb41e1.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6a5a6bfa-8f61-418e-8fc4-0c5d8ddb41e1.json new file mode 100644 index 00000000000..318b4599fa6 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6a5a6bfa-8f61-418e-8fc4-0c5d8ddb41e1.json @@ -0,0 +1,157 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Sources by Size [Filebeat CEF]", + "uiStateJSON": { + "P-11": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "P-13": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "P-2": { + "mapCenter": [ + -0.17578097424708533, + 0 + ], + "mapZoom": 0 + }, + "P-3": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "P-4": { + "mapCenter": [ + -0.17578097424708533, + 0 + ], + "mapZoom": 0 + }, + "P-5": { + "vis": { + "defaultColors": { + "0 - 18,000": "rgb(247,251,255)", + "108,000 - 126,000": "rgb(74,152,201)", + "126,000 - 144,000": "rgb(46,126,188)", + "144,000 - 162,000": "rgb(23,100,171)", + "162,000 - 180,000": "rgb(8,74,145)", + "18,000 - 36,000": "rgb(227,238,249)", + "36,000 - 54,000": "rgb(208,225,242)", + "54,000 - 72,000": "rgb(182,212,233)", + "72,000 - 90,000": "rgb(148,196,223)", + "90,000 - 108,000": "rgb(107,174,214)" + }, + "legendOpen": false + } + }, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Sources", + "field": "source.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destinations", + "field": "destination.domain" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Count" + }, + "schema": "metric", + "type": "count" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Sources by Size [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-6a5a6bfa-8f61-418e-8fc4-0c5d8ddb41e1", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6d82b7f1-5400-43b4-91c0-9f27952d571f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6d82b7f1-5400-43b4-91c0-9f27952d571f.json new file mode 100644 index 00000000000..3e6541034a6 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-6d82b7f1-5400-43b4-91c0-9f27952d571f.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Vendors by Product — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "OS Vendor", + "field": "cef.device.vendor", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "OS Product", + "field": "cef.device.product", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 5 Vendors by Product — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-6d82b7f1-5400-43b4-91c0-9f27952d571f", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MDksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-71a210f8-75b5-412b-b5e7-730d8976988d.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-71a210f8-75b5-412b-b5e7-730d8976988d.json new file mode 100644 index 00000000000..c65a7a49930 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-71a210f8-75b5-412b-b5e7-730d8976988d.json @@ -0,0 +1,140 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Device — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Host\" OR cef.extensions.categoryDeviceGroup:\"/Application\"" + }, + "id": "fd1ffeb6-678e-4163-9421-6a164fd59048", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(254,37,37,1)", + "fill": "0", + "formatter": "number", + "id": "6a10f77d-4e26-4b27-9c19-f1b0029b075b", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "845b9164-65f4-4599-b9cc-8d91b6ba8d83", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "845b9164-65f4-4599-b9cc-8d91b6ba8d83", + "gamma": 0.3, + "id": "59675e84-1a8e-41df-9f63-875109bd795a", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\" " + }, + "id": "d9a580c3-eb83-4d20-a391-0934d7df8837", + "label": "Operating System" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": " cef.extensions.categoryDeviceGroup:\"/IDS/Host\"" + }, + "id": "9ce8be14-6191-4c9a-a679-e3992fdab8d2", + "label": "Host IDS" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Application\"" + }, + "id": "262ecd54-a042-4bfb-b489-d7db8431c36e", + "label": "Application" + } + ], + "split_mode": "filters", + "stacked": "none" + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "92e98952-8e25-472f-abb5-05a7d9b830ea", + "label": "Moving Average by Device HostNames", + "line_width": 1, + "metrics": [ + { + "id": "3df841a9-5997-4a1a-ad8f-69620d23e65b", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "3df841a9-5997-4a1a-ad8f-69620d23e65b", + "gamma": 0.3, + "id": "9765367a-0fc2-45ba-88a8-e87991210edd", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "observer.hostname" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Device — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-71a210f8-75b5-412b-b5e7-730d8976988d", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-74fb53d5-cba8-4bd4-9384-d831e706e0b4.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-74fb53d5-cba8-4bd4-9384-d831e706e0b4.json new file mode 100644 index 00000000000..ce42771b345 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-74fb53d5-cba8-4bd4-9384-d831e706e0b4.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Vendors by Product [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "OS Vendor", + "field": "cef.device.vendor", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "OS Product", + "field": "cef.device.product", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 5 Vendors by Product [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-74fb53d5-cba8-4bd4-9384-d831e706e0b4", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7556bb16-ddfb-4aca-b351-662bbb699093.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7556bb16-ddfb-4aca-b351-662bbb699093.json new file mode 100644 index 00000000000..47a396ead76 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7556bb16-ddfb-4aca-b351-662bbb699093.json @@ -0,0 +1,152 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Outcomes Breakdown [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "failure": "#BF1B00", + "unknown": "#3F2B5B" + }, + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Time", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "event.outcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Time" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "area", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Outcomes Breakdown [Filebeat CEF]", + "type": "area" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-7556bb16-ddfb-4aca-b351-662bbb699093", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-765d9f2a-7f2e-4765-94f3-77d9359ef50f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-765d9f2a-7f2e-4765-94f3-77d9359ef50f.json new file mode 100644 index 00000000000..8933321f0aa --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-765d9f2a-7f2e-4765-94f3-77d9359ef50f.json @@ -0,0 +1,147 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Device Types — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "2fddda5e-d6fc-4581-bbb7-574e1017ae8f" + } + ], + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "gauge_color_rules": [ + { + "id": "3ed9a6b9-fd2e-4e0d-bd83-7ad467b3c8a4" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(211,49,21,1)", + "fill": "0", + "filter": "", + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "e5a48d9d-7834-4da7-8d78-7d4528136b9b", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "sigma": "", + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "id": "78bfdf07-ec02-4dd8-8ff4-b7e250c561c2", + "label": "Firewall" + } + ], + "split_mode": "everything", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostname", + "terms_order_by": null + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(251,158,0,1)", + "fill": 0.5, + "formatter": "number", + "id": "29d6131a-5143-4a64-b597-9538692f0269", + "label": "Top Device Types by Mvg Averages", + "line_width": 1, + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "gamma": 0.3, + "id": "87e21aaa-12eb-4213-bb37-41cb19219240", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "cef.extensions.categoryDeviceType", + "terms_size": "10" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Device Types — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-765d9f2a-7f2e-4765-94f3-77d9359ef50f", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-78e69375-2f26-437f-a9b8-cb8d4bc26641.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-78e69375-2f26-437f-a9b8-cb8d4bc26641.json new file mode 100644 index 00000000000..0158c4fb897 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-78e69375-2f26-437f-a9b8-cb8d4bc26641.json @@ -0,0 +1,104 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Endpoint - Average EPS [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "bar_color_rules": [ + { + "id": "ce9549a0-3af0-4070-b169-4b6d145d4c39" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "event.dataset:\"cef.log\"" + }, + "gauge_color_rules": [ + { + "id": "03a2fd72-fc9c-4582-9133-20af36217180" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "94161c6c-4f48-4beb-9d78-f79f29c02a34", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "b4373ffd-9660-4206-afd6-d4867ac7dbdf", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "type": "count" + }, + { + "field": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "id": "89f8286e-4aec-4cb4-83ad-b139692edf3d", + "type": "cumulative_sum" + }, + { + "field": "89f8286e-4aec-4cb4-83ad-b139692edf3d", + "id": "1df39e5f-3e98-4ed7-ab08-47f3ca2ee915", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "1df39e5f-3e98-4ed7-ab08-47f3ca2ee915", + "gamma": 0.3, + "id": "f46a6e6e-444f-4c7e-b5eb-e1a59568f2eb", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "offset_time": "1m", + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "Endpoint - Average EPS [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-78e69375-2f26-437f-a9b8-cb8d4bc26641", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7cd4866d-4085-44d1-ad12-45559570a176.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7cd4866d-4085-44d1-ad12-45559570a176.json new file mode 100644 index 00000000000..607e773380c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7cd4866d-4085-44d1-ad12-45559570a176.json @@ -0,0 +1,132 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top Destinations by Traffic Size [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 18k": "rgb(247,251,255)", + "108k - 126k": "rgb(74,152,201)", + "126k - 144k": "rgb(46,126,188)", + "144k - 162k": "rgb(23,100,171)", + "162k - 180k": "rgb(8,74,145)", + "18k - 36k": "rgb(227,238,249)", + "36k - 54k": "rgb(208,225,242)", + "54k - 72k": "rgb(182,212,233)", + "72k - 90k": "rgb(148,196,223)", + "90k - 108k": "rgb(107,174,214)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "2", + "params": { + "filters": [ + { + "input": { + "language": "lucene", + "query": "deviceDirection:\"0\"" + }, + "label": "Inbound" + }, + { + "input": { + "language": "lucene", + "query": "deviceDirection:\"1\"" + }, + "label": "Outbound" + } + ] + }, + "schema": "segment", + "type": "filters" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Blues", + "colorsNumber": 10, + "colorsRange": [ + { + "from": 0, + "to": null + } + ], + "enableHover": true, + "invertColors": false, + "legendPosition": "top", + "legendSize": "auto", + "percentageMode": false, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "Top Destinations by Traffic Size [Filebeat CEF]", + "type": "heatmap" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-7cd4866d-4085-44d1-ad12-45559570a176", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0.json new file mode 100644 index 00000000000..666aaecf890 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "query_string": { + "query": "*" + } + } + } + }, + "title": " Dashboard Navigation — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "markdown": "[Network Overview](#/dashboard/cef-346ac50b-0900-464d-aad6-007756a4e215) | [Network Suspicious Activity](#/dashboard/cef-6d77b0a1-7b83-4c32-bbc5-cb1f6a4ee634) | [Endpoint Overview](#dashboard/cef-1ef1282f-382b-4bdd-9cb1-810b8f083e74) | [Endpoint OS Activity](#/dashboard/cef-110da36b-6544-4507-9ae5-591d44bd2333) | [Microsoft DNS Overview](#/dashboard/cef-b16c980c-21ad-47ca-8103-7ce949ddc87a)" + }, + "title": " Dashboard Navigation — ArcSight [Filebeat CEF]", + "type": "markdown" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-7ce3dc59-a4ec-4748-a5f4-7a35a4a055e0", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-81cc567b-7ef9-44f8-9fc6-9fd98f8d8a3d.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-81cc567b-7ef9-44f8-9fc6-9fd98f8d8a3d.json new file mode 100644 index 00000000000..f64d733e6e2 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-81cc567b-7ef9-44f8-9fc6-9fd98f8d8a3d.json @@ -0,0 +1,202 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Source Users by Event Type and Destination Users — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Users", + "field": "source.user.name", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Types", + "field": "cef.extensions.categoryBehavior" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination User Names", + "field": "destination.user.name" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Source Users" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Event Types" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "stacked", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + }, + { + "data": { + "id": "4", + "label": "Destination User Names" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "stacked", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "" + }, + "type": "value" + } + ] + }, + "title": "Source Users by Event Type and Destination Users — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-81cc567b-7ef9-44f8-9fc6-9fd98f8d8a3d", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-82b8ee84-62bb-47b2-a865-a80caf4cb13a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-82b8ee84-62bb-47b2-a865-a80caf4cb13a.json new file mode 100644 index 00000000000..2679cb31752 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-82b8ee84-62bb-47b2-a865-a80caf4cb13a.json @@ -0,0 +1,120 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Devices by Bandwidth [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Device", + "field": "observer.hostname", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Source(s)", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Destination(s)", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Bandwidth (Incoming)", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Bandwidth (Outgoing)", + "field": "destination.bytes" + }, + "schema": "metric", + "type": "sum" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Devices by Bandwidth [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-82b8ee84-62bb-47b2-a865-a80caf4cb13a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8613b170-e504-4a37-b723-baeb94470380.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8613b170-e504-4a37-b723-baeb94470380.json new file mode 100644 index 00000000000..92f48bc3eb8 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8613b170-e504-4a37-b723-baeb94470380.json @@ -0,0 +1,181 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Event Types by Size [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "Count": "#64B0C8", + "Total (Bytes)": "#E24D42" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Type", + "field": "cef.device.event_class_id", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Total (Bytes)", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "rotate": 75, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Event Type" + }, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + }, + "valueAxis": null + }, + "legendPosition": "right", + "legendSize": "auto", + "orderBucketsBySum": false, + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Total (Bytes)" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 3, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "Total (Bytes)" + }, + "type": "value" + } + ] + }, + "title": "Event Types by Size [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8613b170-e504-4a37-b723-baeb94470380", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MTksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8922e448-709a-4040-b522-dc520a7cf711.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8922e448-709a-4040-b522-dc520a7cf711.json new file mode 100644 index 00000000000..78e8e396ace --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8922e448-709a-4040-b522-dc520a7cf711.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Sources [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Hosts", + "field": "source.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Sources [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8922e448-709a-4040-b522-dc520a7cf711", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-89236ffe-9484-472f-8e70-a58db4e2feb3.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-89236ffe-9484-472f-8e70-a58db4e2feb3.json new file mode 100644 index 00000000000..4de8b71dca6 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-89236ffe-9484-472f-8e70-a58db4e2feb3.json @@ -0,0 +1,181 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Unique Destinations and Ports by Source — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Source Addresses" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Destination Addresses" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Destination Ports" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "stacked", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Destination Addresses" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Destination Ports" + }, + "type": "value" + } + ] + }, + "title": "Unique Destinations and Ports by Source — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-89236ffe-9484-472f-8e70-a58db4e2feb3", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8d4f3630-04f0-4b40-8d56-5b989b471370.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8d4f3630-04f0-4b40-8d56-5b989b471370.json new file mode 100644 index 00000000000..8b6d8364ad9 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8d4f3630-04f0-4b40-8d56-5b989b471370.json @@ -0,0 +1,181 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Event Types by Size — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "Count": "#64B0C8", + "Total (Bytes)": "#E24D42" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Type", + "field": "cef.device.event_class_id", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Total (Bytes)", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "rotate": 75, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Event Type" + }, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + }, + "valueAxis": null + }, + "legendPosition": "right", + "legendSize": "auto", + "orderBucketsBySum": false, + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Total (Bytes)" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 3, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "Total (Bytes)" + }, + "type": "value" + } + ] + }, + "title": "Event Types by Size — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8d4f3630-04f0-4b40-8d56-5b989b471370", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8d5ffb86-ef60-464e-82d9-d6f6de11cdd7.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8d5ffb86-ef60-464e-82d9-d6f6de11cdd7.json new file mode 100644 index 00000000000..9e624b24436 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8d5ffb86-ef60-464e-82d9-d6f6de11cdd7.json @@ -0,0 +1,118 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Outcomes [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "event.dataset:\"cef.log\"" + }, + "id": "74716d29-91c6-4095-bc7d-7f6700f12b1f", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": "0", + "formatter": "number", + "hide_in_legend": 0, + "id": "932c5de4-f841-4f27-99e4-60d95d3aa16c", + "label": "Event Outcomes", + "line_width": "3", + "metrics": [ + { + "id": "4c263b6d-8117-43c6-b83f-5c4145f43cfc", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Failure\"" + }, + "id": "94371b84-a7aa-4824-b4d1-217ecbe725a5", + "label": "Failure" + }, + { + "color": "rgba(104,188,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Success\"" + }, + "id": "31564794-9278-4f2e-bb20-557f5cfbea79", + "label": "Success" + }, + { + "color": "rgba(251,158,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Attempt\"" + }, + "id": "10c0f919-0853-41b5-94b4-2e39932e7aa0", + "label": "Attempt" + } + ], + "split_mode": "filters", + "stacked": "none", + "terms_field": "event.outcome", + "terms_size": "3" + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(104,182,204,1)", + "fill": 0.5, + "formatter": "number", + "id": "c9eca9d0-c2e0-45e6-a3ce-f158c40fdd74", + "label": "Event Count", + "line_width": 1, + "metrics": [ + { + "id": "6d8513ca-cc72-4b27-91b6-6b689558cdcb", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Outcomes [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8d5ffb86-ef60-464e-82d9-d6f6de11cdd7", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8ed5df68-1651-4e52-8c89-997109ddee63.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8ed5df68-1651-4e52-8c89-997109ddee63.json new file mode 100644 index 00000000000..2d9bf685cb6 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8ed5df68-1651-4e52-8c89-997109ddee63.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "query_string": { + "query": "*" + } + } + } + }, + "title": " Dashboard Navigation — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "markdown": "[Network Overview](#/dashboard/cef-346ac50b-0900-464d-aad6-007756a4e215) | [Network Suspicious Activity](#/dashboard/cef-6d77b0a1-7b83-4c32-bbc5-cb1f6a4ee634) | [Endpoint Overview](#dashboard/cef-1ef1282f-382b-4bdd-9cb1-810b8f083e74) | [Endpoint OS Activity](#/dashboard/cef-110da36b-6544-4507-9ae5-591d44bd2333) | [Microsoft DNS Overview](#/dashboard/cef-b16c980c-21ad-47ca-8103-7ce949ddc87a)" + }, + "title": " Dashboard Navigation — ArcSight [Filebeat CEF]", + "type": "markdown" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8ed5df68-1651-4e52-8c89-997109ddee63", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8f26b7a6-7ae0-4a01-9c0e-b7a0e0302fd7.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8f26b7a6-7ae0-4a01-9c0e-b7a0e0302fd7.json new file mode 100644 index 00000000000..4f5a405e528 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-8f26b7a6-7ae0-4a01-9c0e-b7a0e0302fd7.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Addresses [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Source Addresses [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-8f26b7a6-7ae0-4a01-9c0e-b7a0e0302fd7", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-95877a19-0c82-4194-ad67-b311c458c6f4.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-95877a19-0c82-4194-ad67-b311c458c6f4.json new file mode 100644 index 00000000000..116e9e90c21 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-95877a19-0c82-4194-ad67-b311c458c6f4.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destinations by Size [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destinations", + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Sources", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Count" + }, + "schema": "metric", + "type": "count" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Destinations by Size [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-95877a19-0c82-4194-ad67-b311c458c6f4", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-71d2b40c-bcc3-48cc-8f31-9134ec3a681e", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-97a3e327-9b8b-4a1b-91ad-6c93c90d0193.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-97a3e327-9b8b-4a1b-91ad-6c93c90d0193.json new file mode 100644 index 00000000000..a642e3b7ca0 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-97a3e327-9b8b-4a1b-91ad-6c93c90d0193.json @@ -0,0 +1,204 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Events by Source and Destination Users — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Timestamp", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Users", + "field": "source.user.name" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Timestamp" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Event Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Source Users" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 3, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + }, + { + "data": { + "id": "4", + "label": "Destination Users" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 3, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Event Count" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "square root" + }, + "show": true, + "style": {}, + "title": { + "text": "" + }, + "type": "value" + } + ] + }, + "title": "Events by Source and Destination Users — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-97a3e327-9b8b-4a1b-91ad-6c93c90d0193", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-98511d76-8c58-4c0b-8723-6b495a0fd56e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-98511d76-8c58-4c0b-8723-6b495a0fd56e.json new file mode 100644 index 00000000000..45d9ba01993 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-98511d76-8c58-4c0b-8723-6b495a0fd56e.json @@ -0,0 +1,115 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Severity — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "bar_color_rules": [ + { + "id": "0ca18a89-9c81-4bee-835a-85e6103aec37" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "hide_last_value_indicator": true, + "id": "c39a76e5-f613-41a9-8335-c442747791e0", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "0.0[0]a", + "id": "da3b92b4-2c24-473b-9102-fb5a343a96d9", + "label": "Event by Severities", + "line_width": 1, + "metrics": [ + { + "id": "0d189776-3f7c-4a92-95b1-73c379a341fc", + "type": "count" + }, + { + "field": "0d189776-3f7c-4a92-95b1-73c379a341fc", + "id": "1b1c931c-a09b-4980-af81-6f9c3db56401", + "sigma": "", + "type": "sum_bucket" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(104,204,202,1)", + "filter": { + "language": "lucene", + "query": "severity:\"Low\" OR severity:\"0\"" + }, + "id": "ebe970ac-5cc9-4c4a-af60-82affafc667c", + "label": "LOW" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "severity:\"Medium\"" + }, + "id": "0c4ff16a-b53d-4ce4-af76-d6b74d8788db", + "label": "MEDIUM" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "severity:\"High\"" + }, + "id": "e142c55b-6ee5-416a-8bd3-d10398044864", + "label": "HIGH" + }, + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "severity:\"Very-High\"" + }, + "id": "4b05b562-c419-4214-b814-d4c242251521", + "label": "VERY HIGH" + } + ], + "split_mode": "filters", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "top_n", + "use_kibana_indexes": false + }, + "title": "Events by Severity — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-98511d76-8c58-4c0b-8723-6b495a0fd56e", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9b0b7c11-1eaf-466c-8dab-4e3299d7091f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9b0b7c11-1eaf-466c-8dab-4e3299d7091f.json new file mode 100644 index 00000000000..7ddecfdfd41 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9b0b7c11-1eaf-466c-8dab-4e3299d7091f.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Sources by Destination Ports — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Source Address", + "field": "source.ip", + "order": "desc", + "orderBy": "2", + "size": 5 + }, + "schema": "bucket", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 5 Sources by Destination Ports — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-9b0b7c11-1eaf-466c-8dab-4e3299d7091f", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MjksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9cebd9ff-0094-40f3-bed7-45dea62a5279.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9cebd9ff-0094-40f3-bed7-45dea62a5279.json new file mode 100644 index 00000000000..309ca4d97d1 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9cebd9ff-0094-40f3-bed7-45dea62a5279.json @@ -0,0 +1,140 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Device [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "event.dataset:\"cef.log\"" + }, + "id": "fd1ffeb6-678e-4163-9421-6a164fd59048", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(254,37,37,1)", + "fill": "0", + "formatter": "number", + "id": "6a10f77d-4e26-4b27-9c19-f1b0029b075b", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "845b9164-65f4-4599-b9cc-8d91b6ba8d83", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "845b9164-65f4-4599-b9cc-8d91b6ba8d83", + "gamma": 0.3, + "id": "59675e84-1a8e-41df-9f63-875109bd795a", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\" " + }, + "id": "d9a580c3-eb83-4d20-a391-0934d7df8837", + "label": "Operating System" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": " cef.extensions.categoryDeviceGroup:\"/IDS/Host\"" + }, + "id": "9ce8be14-6191-4c9a-a679-e3992fdab8d2", + "label": "Host IDS" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Application\"" + }, + "id": "262ecd54-a042-4bfb-b489-d7db8431c36e", + "label": "Application" + } + ], + "split_mode": "filters", + "stacked": "none" + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "92e98952-8e25-472f-abb5-05a7d9b830ea", + "label": "Moving Average by Device HostNames", + "line_width": 1, + "metrics": [ + { + "id": "3df841a9-5997-4a1a-ad8f-69620d23e65b", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "3df841a9-5997-4a1a-ad8f-69620d23e65b", + "gamma": 0.3, + "id": "9765367a-0fc2-45ba-88a8-e87991210edd", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "observer.hostname" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Device [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-9cebd9ff-0094-40f3-bed7-45dea62a5279", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9d5b10e7-e2be-4913-8f55-27bb72a0f787.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9d5b10e7-e2be-4913-8f55-27bb72a0f787.json new file mode 100644 index 00000000000..836cf8c5f16 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9d5b10e7-e2be-4913-8f55-27bb72a0f787.json @@ -0,0 +1,140 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Source Addresses — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "a0bf5a1d-8ebf-49d4-a347-738a6ce20562" + } + ], + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\" " + }, + "gauge_color_rules": [ + { + "id": "42f84a0a-ee13-4ca8-b61d-3de482ae4ab0" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(211,49,21,1)", + "fill": "0", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\" " + }, + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "117fde19-e227-4fcb-8019-e82e6677c340", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "sigma": "", + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostmessage", + "terms_order_by": null, + "value_template": "{{value}}" + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(104,188,0,1)", + "fill": "0.5", + "formatter": "number", + "id": "3ffe652e-43c2-4a1d-ad8a-f7ab10f09f2b", + "label": "Top Source Addresses", + "line_width": "0", + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "gamma": 0.3, + "id": "b753ad38-c3ed-4463-8f6d-176f4d477897", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "source.ip", + "terms_size": "10" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Source Addresses — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-9d5b10e7-e2be-4913-8f55-27bb72a0f787", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9e73269e-bb91-45d0-9b5f-2c2ed32c451f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9e73269e-bb91-45d0-9b5f-2c2ed32c451f.json new file mode 100644 index 00000000000..f188b07d81c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-9e73269e-bb91-45d0-9b5f-2c2ed32c451f.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Sources by Destination Ports [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Source Address", + "field": "source.ip", + "order": "desc", + "orderBy": "2", + "size": 5 + }, + "schema": "bucket", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 5 Sources by Destination Ports [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-9e73269e-bb91-45d0-9b5f-2c2ed32c451f", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a09db928-f867-46cf-85f9-1d0a29afa903.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a09db928-f867-46cf-85f9-1d0a29afa903.json new file mode 100644 index 00000000000..8d1f3d74eb9 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a09db928-f867-46cf-85f9-1d0a29afa903.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Users — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Users", + "field": "source.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Source Users — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-a09db928-f867-46cf-85f9-1d0a29afa903", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a1aec4bd-be82-46bf-a35b-23cebee7b218.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a1aec4bd-be82-46bf-a35b-23cebee7b218.json new file mode 100644 index 00000000000..05b5d29bd02 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a1aec4bd-be82-46bf-a35b-23cebee7b218.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Users by Destination Users [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Users", + "field": "source.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "bottom", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 10 Source Users by Destination Users [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-a1aec4bd-be82-46bf-a35b-23cebee7b218", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a1c8e760-dcca-4ae8-87d5-00461a179a5a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a1c8e760-dcca-4ae8-87d5-00461a179a5a.json new file mode 100644 index 00000000000..c5211110d10 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a1c8e760-dcca-4ae8-87d5-00461a179a5a.json @@ -0,0 +1,135 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Device Metrics Overview — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Sources", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destinations", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "12", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Device Metrics Overview — ArcSight [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-a1c8e760-dcca-4ae8-87d5-00461a179a5a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a29c8241-2598-4964-b800-0e298e9d2ad8.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a29c8241-2598-4964-b800-0e298e9d2ad8.json new file mode 100644 index 00000000000..4086fdec67b --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a29c8241-2598-4964-b800-0e298e9d2ad8.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Sources — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Hosts", + "field": "source.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Sources — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-a29c8241-2598-4964-b800-0e298e9d2ad8", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a35b83af-0f72-4825-922f-76a71022c2fb.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a35b83af-0f72-4825-922f-76a71022c2fb.json new file mode 100644 index 00000000000..5222f2cae63 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-a35b83af-0f72-4825-922f-76a71022c2fb.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Sources by Destination Addresses [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Source Address", + "field": "source.ip", + "order": "desc", + "orderBy": "2", + "size": 5 + }, + "schema": "bucket", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 5 Sources by Destination Addresses [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-a35b83af-0f72-4825-922f-76a71022c2fb", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-b85d3a62-8dc5-45f8-a561-b0fb5d2d7a5e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-b85d3a62-8dc5-45f8-a561-b0fb5d2d7a5e.json new file mode 100644 index 00000000000..aed7785efed --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-b85d3a62-8dc5-45f8-a561-b0fb5d2d7a5e.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Destination Ports by Outcomes — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "destination.port: Descending" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "defaultYExtents": true, + "mode": "normal", + "setYExtents": false, + "type": "square root" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Destination Ports by Outcomes — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-b85d3a62-8dc5-45f8-a561-b0fb5d2d7a5e", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-b8f5c70f-5dfa-4cdc-8bbd-7cefcee4c902.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-b8f5c70f-5dfa-4cdc-8bbd-7cefcee4c902.json new file mode 100644 index 00000000000..a4544981833 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-b8f5c70f-5dfa-4cdc-8bbd-7cefcee4c902.json @@ -0,0 +1,124 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Direction — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "id": "be556a57-cd1c-496c-8714-0bd210947c85", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "bar", + "color": "#68BC00", + "fill": "0.2", + "filter": { + "language": "lucene", + "query": "device" + }, + "formatter": "number", + "id": "9aae7344-9de9-4378-b21d-296cb964f93b", + "label": "Inbound Requests", + "line_width": 1, + "metrics": [ + { + "id": "1cd0b964-45cf-408e-a7e4-e26955f8a3b0", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(0,156,224,1)", + "filter": { + "language": "lucene", + "query": "deviceDirection:\"0\"" + }, + "id": "f860f6e0-fbd4-4949-8046-6300322dfe84", + "label": "Inbound Requests" + } + ], + "split_mode": "filters", + "stacked": "none" + }, + { + "axis_position": "right", + "chart_type": "bar", + "color": "#68BC00", + "fill": "0.2", + "formatter": "number", + "id": "ed1abe18-e01b-4202-9db4-06fda10692e0", + "label": "Outbound Requests", + "line_width": 1, + "metrics": [ + { + "id": "cfbcfc79-394b-4ec0-a2c2-7a47177d6469", + "type": "count" + }, + { + "id": "6bc37118-ddac-41ec-85b3-9db7e1b3636b", + "script": "params.outbound \u003e 0 ? params.outbound * -1 : 0", + "type": "calculation", + "variables": [ + { + "field": "cfbcfc79-394b-4ec0-a2c2-7a47177d6469", + "id": "f73f4f22-03d5-446a-b031-04eee531e3cc", + "name": "outbound" + } + ] + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(211,49,21,1)", + "filter": { + "language": "lucene", + "query": "deviceDirection:\"1\"" + }, + "id": "a9c50e1b-8f11-4bc2-9077-bb8870ed0b62", + "label": "Outbound Requests" + } + ], + "split_mode": "filters", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Direction — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-b8f5c70f-5dfa-4cdc-8bbd-7cefcee4c902", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0MzksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bc37e59b-a9b4-4de1-8f24-2706b65b888a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bc37e59b-a9b4-4de1-8f24-2706b65b888a.json new file mode 100644 index 00000000000..0ddd38f5a3b --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bc37e59b-a9b4-4de1-8f24-2706b65b888a.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "query_string": { + "query": "*" + } + } + } + }, + "title": " Dashboard Navigation [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "markdown": "[Network Overview](#/dashboard/cef-8d7c3098-7b2c-4d90-865b-75da6bfb29b6) | [Network Suspicious Activity](#/dashboard/cef-2745d8ff-e261-428b-a406-15ee77c8a3dd) | [Endpoint Overview](#dashboard/cef-c5ea76fb-5d0b-462d-922b-978ccd1f0f5c) | [Endpoint Activity](#/dashboard/cef-87e7074a-0764-4cdd-a39d-ec1be32c292f) | [Microsoft DNS Overview](#/dashboard/cef-244684fd-7be2-49fb-a817-e51cec267bf1)" + }, + "title": " Dashboard Navigation [Filebeat CEF]", + "type": "markdown" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-bc37e59b-a9b4-4de1-8f24-2706b65b888a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bd8f3914-cf95-4451-bd26-482130922a4f.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bd8f3914-cf95-4451-bd26-482130922a4f.json new file mode 100644 index 00000000000..42f796ae614 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bd8f3914-cf95-4451-bd26-482130922a4f.json @@ -0,0 +1,111 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "DNS - Event Throughput — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color_rules": [ + { + "id": "3eadd451-5033-423f-88e3-814cc5e50b50" + } + ], + "bar_color_rules": [ + { + "id": "fa374805-d1ca-4261-b723-9b482a7dd43a" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "gauge_color_rules": [ + { + "gauge": null, + "id": "4d957654-cc7e-4ef3-8b29-61c0aeadd51a", + "value": 0 + } + ], + "gauge_inner_width": 10, + "gauge_max": "", + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "73968651-c41e-473e-a153-a025f49d1a1b", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "90d7621e-3265-4fe8-8882-8df9605ea659", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "type": "count" + }, + { + "field": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "id": "cf3e6b1c-4136-4868-913e-0e82d88a8c9c", + "type": "cumulative_sum" + }, + { + "field": "cf3e6b1c-4136-4868-913e-0e82d88a8c9c", + "id": "0e407985-9ae4-4c1f-bb0e-16cd9bef7611", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "0e407985-9ae4-4c1f-bb0e-16cd9bef7611", + "gamma": 0.3, + "id": "48026f85-83c8-40e6-aff4-71f3bd6c77c9", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "DNS - Event Throughput — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-bd8f3914-cf95-4451-bd26-482130922a4f", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-beb62b4c-dbe4-4f72-8a70-5a102a9ae139.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-beb62b4c-dbe4-4f72-8a70-5a102a9ae139.json new file mode 100644 index 00000000000..9df5ba844a2 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-beb62b4c-dbe4-4f72-8a70-5a102a9ae139.json @@ -0,0 +1,147 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Device Types [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "2fddda5e-d6fc-4581-bbb7-574e1017ae8f" + } + ], + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "gauge_color_rules": [ + { + "id": "3ed9a6b9-fd2e-4e0d-bd83-7ad467b3c8a4" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(211,49,21,1)", + "fill": "0", + "filter": "", + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "e5a48d9d-7834-4da7-8d78-7d4528136b9b", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "sigma": "", + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "id": "78bfdf07-ec02-4dd8-8ff4-b7e250c561c2", + "label": "Firewall" + } + ], + "split_mode": "everything", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostname", + "terms_order_by": null + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(251,158,0,1)", + "fill": 0.5, + "formatter": "number", + "id": "29d6131a-5143-4a64-b597-9538692f0269", + "label": "Top Device Types by Mvg Averages", + "line_width": 1, + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "gamma": 0.3, + "id": "87e21aaa-12eb-4213-bb37-41cb19219240", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "cef.extensions.categoryDeviceType", + "terms_size": "10" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Device Types [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-beb62b4c-dbe4-4f72-8a70-5a102a9ae139", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bfc6d482-4f8e-40f8-b4ee-10ba19675d42.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bfc6d482-4f8e-40f8-b4ee-10ba19675d42.json new file mode 100644 index 00000000000..d3aedafbc75 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-bfc6d482-4f8e-40f8-b4ee-10ba19675d42.json @@ -0,0 +1,108 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Size [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "id": "6e634117-6b30-411c-b74c-75510befe42f", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "filter": { + "language": "lucene", + "query": "deviceDirection:\"0\"" + }, + "formatter": "bytes", + "id": "28b1fb5b-0f16-4519-b901-4dd2dcc39915", + "label": "Inbound Bytes", + "line_width": "2", + "metrics": [ + { + "field": "source.bytes", + "id": "f613f33f-6459-4e46-a3a0-c36c48c46b2e", + "type": "sum" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "filter", + "stacked": "none" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": 0.5, + "filter": { + "language": "lucene", + "query": "deviceDirection:\"1\"" + }, + "formatter": "bytes", + "id": "5a5c2529-4990-4006-b039-c94069ff6b7e", + "label": "Outbound Bytes", + "line_width": "2", + "metrics": [ + { + "field": "source.bytes", + "id": "b69501e7-56d5-4c38-81d1-34d778c81e11", + "type": "sum" + }, + { + "id": "0aaab374-5845-44ab-94f5-ac4fab25c287", + "script": "params.outbound_bytes \u003e= 0 ? params.outbound_bytes * -1 : 0", + "type": "calculation", + "variables": [ + { + "field": "b69501e7-56d5-4c38-81d1-34d778c81e11", + "id": "23b8c41c-0e98-4ace-8bca-3593e46cd955", + "name": "outbound_bytes" + } + ] + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "filter", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Size [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-bfc6d482-4f8e-40f8-b4ee-10ba19675d42", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c0cf5f53-3ddc-463f-aee7-71d7219a36c5.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c0cf5f53-3ddc-463f-aee7-71d7219a36c5.json new file mode 100644 index 00000000000..1d68b5639be --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c0cf5f53-3ddc-463f-aee7-71d7219a36c5.json @@ -0,0 +1,151 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Outcomes by Device Type — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "/Failure": "#BF1B00" + }, + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "cef.extensions.categoryDeviceType", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 200 + }, + "position": "left", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "cef.extensions.categoryDeviceType: Descending" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": true, + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": true, + "rotate": 75, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "bottom", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Outcomes by Device Type — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-c0cf5f53-3ddc-463f-aee7-71d7219a36c5", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c3b99882-cd00-4bc1-a64f-d7d7ac3ad292.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c3b99882-cd00-4bc1-a64f-d7d7ac3ad292.json new file mode 100644 index 00000000000..141012fd463 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c3b99882-cd00-4bc1-a64f-d7d7ac3ad292.json @@ -0,0 +1,111 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "DNS - Event Throughput [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color_rules": [ + { + "id": "3eadd451-5033-423f-88e3-814cc5e50b50" + } + ], + "bar_color_rules": [ + { + "id": "fa374805-d1ca-4261-b723-9b482a7dd43a" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "gauge_color_rules": [ + { + "gauge": null, + "id": "4d957654-cc7e-4ef3-8b29-61c0aeadd51a", + "value": 0 + } + ], + "gauge_inner_width": 10, + "gauge_max": "", + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "73968651-c41e-473e-a153-a025f49d1a1b", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "90d7621e-3265-4fe8-8882-8df9605ea659", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "type": "count" + }, + { + "field": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "id": "cf3e6b1c-4136-4868-913e-0e82d88a8c9c", + "type": "cumulative_sum" + }, + { + "field": "cf3e6b1c-4136-4868-913e-0e82d88a8c9c", + "id": "0e407985-9ae4-4c1f-bb0e-16cd9bef7611", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "0e407985-9ae4-4c1f-bb0e-16cd9bef7611", + "gamma": 0.3, + "id": "48026f85-83c8-40e6-aff4-71f3bd6c77c9", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "DNS - Event Throughput [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-c3b99882-cd00-4bc1-a64f-d7d7ac3ad292", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c59a4724-de59-4bdf-ba90-da482c6db8ad.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c59a4724-de59-4bdf-ba90-da482c6db8ad.json new file mode 100644 index 00000000000..85738f3c818 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c59a4724-de59-4bdf-ba90-da482c6db8ad.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Sources by Destination Addresses — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Source Address", + "field": "source.ip", + "order": "desc", + "orderBy": "2", + "size": 5 + }, + "schema": "bucket", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 5 Sources by Destination Addresses — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-c59a4724-de59-4bdf-ba90-da482c6db8ad", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c77b16c9-031b-45ec-af29-b8f116c610ba.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c77b16c9-031b-45ec-af29-b8f116c610ba.json new file mode 100644 index 00000000000..f8e33d6ada2 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-c77b16c9-031b-45ec-af29-b8f116c610ba.json @@ -0,0 +1,118 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Outcomes — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Operating System\"" + }, + "id": "74716d29-91c6-4095-bc7d-7f6700f12b1f", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": "0", + "formatter": "number", + "hide_in_legend": 0, + "id": "932c5de4-f841-4f27-99e4-60d95d3aa16c", + "label": "Event Outcomes", + "line_width": "3", + "metrics": [ + { + "id": "4c263b6d-8117-43c6-b83f-5c4145f43cfc", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Failure\"" + }, + "id": "94371b84-a7aa-4824-b4d1-217ecbe725a5", + "label": "Failure" + }, + { + "color": "rgba(104,188,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Success\"" + }, + "id": "31564794-9278-4f2e-bb20-557f5cfbea79", + "label": "Success" + }, + { + "color": "rgba(251,158,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryOutcome:\"/Attempt\"" + }, + "id": "10c0f919-0853-41b5-94b4-2e39932e7aa0", + "label": "Attempt" + } + ], + "split_mode": "filters", + "stacked": "none", + "terms_field": "cef.extensions.categoryOutcome", + "terms_size": "3" + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(104,182,204,1)", + "fill": 0.5, + "formatter": "number", + "id": "c9eca9d0-c2e0-45e6-a3ce-f158c40fdd74", + "label": "Event Count", + "line_width": 1, + "metrics": [ + { + "id": "6d8513ca-cc72-4b27-91b6-6b689558cdcb", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Outcomes — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-c77b16c9-031b-45ec-af29-b8f116c610ba", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d1332acb-7ace-4ee6-96d7-3a154a0405c7.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d1332acb-7ace-4ee6-96d7-3a154a0405c7.json new file mode 100644 index 00000000000..11030cfa040 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d1332acb-7ace-4ee6-96d7-3a154a0405c7.json @@ -0,0 +1,181 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Unique Destinations and Ports by Source [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Source Addresses" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Destination Addresses" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Destination Ports" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "stacked", + "show": true, + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-2" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Destination Addresses" + }, + "type": "value" + }, + { + "id": "ValueAxis-2", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "RightAxis-1", + "position": "right", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Destination Ports" + }, + "type": "value" + } + ] + }, + "title": "Unique Destinations and Ports by Source [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d1332acb-7ace-4ee6-96d7-3a154a0405c7", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d134c1d0-2493-41ea-9ba7-c449a8ec0a6c.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d134c1d0-2493-41ea-9ba7-c449a8ec0a6c.json new file mode 100644 index 00000000000..6548036edbc --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d134c1d0-2493-41ea-9ba7-c449a8ec0a6c.json @@ -0,0 +1,157 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Sources by Size — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "P-11": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "P-13": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "P-2": { + "mapCenter": [ + -0.17578097424708533, + 0 + ], + "mapZoom": 0 + }, + "P-3": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "P-4": { + "mapCenter": [ + -0.17578097424708533, + 0 + ], + "mapZoom": 0 + }, + "P-5": { + "vis": { + "defaultColors": { + "0 - 18,000": "rgb(247,251,255)", + "108,000 - 126,000": "rgb(74,152,201)", + "126,000 - 144,000": "rgb(46,126,188)", + "144,000 - 162,000": "rgb(23,100,171)", + "162,000 - 180,000": "rgb(8,74,145)", + "18,000 - 36,000": "rgb(227,238,249)", + "36,000 - 54,000": "rgb(208,225,242)", + "54,000 - 72,000": "rgb(182,212,233)", + "72,000 - 90,000": "rgb(148,196,223)", + "90,000 - 108,000": "rgb(107,174,214)" + }, + "legendOpen": false + } + }, + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Sources", + "field": "source.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Bytes", + "field": "source.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destinations", + "field": "destination.domain" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Count" + }, + "schema": "metric", + "type": "count" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Sources by Size — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d134c1d0-2493-41ea-9ba7-c449a8ec0a6c", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-721d1d17-9c3a-4002-9f23-d51a12604d41", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NDksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d24ad888-a5cf-4b17-ac08-ae13d4cea23e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d24ad888-a5cf-4b17-ac08-ae13d4cea23e.json new file mode 100644 index 00000000000..e49ed4e23b8 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d24ad888-a5cf-4b17-ac08-ae13d4cea23e.json @@ -0,0 +1,110 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Devices by Outcome — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0% - 17%": "rgb(255,255,204)", + "17% - 34%": "rgb(255,230,146)", + "34% - 50%": "rgb(254,191,90)", + "50% - 67%": "rgb(253,141,60)", + "67% - 84%": "rgb(244,61,37)", + "84% - 100%": "rgb(202,8,35)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Device Host Names", + "field": "observer.hostname", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Yellow to Red", + "colorsNumber": 6, + "colorsRange": [], + "enableHover": true, + "invertColors": false, + "legendPosition": "right", + "legendSize": "auto", + "percentageMode": true, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "Top 10 Devices by Outcome — ArcSight [Filebeat CEF]", + "type": "heatmap" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d24ad888-a5cf-4b17-ac08-ae13d4cea23e", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d2f2ec63-b44f-4b7c-8a48-9d457d974192.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d2f2ec63-b44f-4b7c-8a48-9d457d974192.json new file mode 100644 index 00000000000..e2c42cb44c5 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d2f2ec63-b44f-4b7c-8a48-9d457d974192.json @@ -0,0 +1,109 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Countries by Events [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total Events" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Country", + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Source Countries by Events [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d2f2ec63-b44f-4b7c-8a48-9d457d974192", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d618f6ab-e55c-42e9-b71e-f65e45630795.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d618f6ab-e55c-42e9-b71e-f65e45630795.json new file mode 100644 index 00000000000..523e20aaa35 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d618f6ab-e55c-42e9-b71e-f65e45630795.json @@ -0,0 +1,111 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Network - Event Throughput — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color_rules": [ + { + "id": "3eadd451-5033-423f-88e3-814cc5e50b50" + } + ], + "bar_color_rules": [ + { + "id": "8d4596c5-49ad-429b-af54-5451b1c2e8d4" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\" " + }, + "gauge_color_rules": [ + { + "gauge": null, + "id": "4d957654-cc7e-4ef3-8b29-61c0aeadd51a", + "value": 0 + } + ], + "gauge_inner_width": 10, + "gauge_max": "", + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "73968651-c41e-473e-a153-a025f49d1a1b", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "90d7621e-3265-4fe8-8882-8df9605ea659", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "type": "count" + }, + { + "field": "ba1830b9-9ce3-4bf1-8f4d-f7478b7f1bba", + "id": "ca3a65d0-9f3d-42a9-9f4e-16f9e24cba19", + "type": "cumulative_sum" + }, + { + "field": "ca3a65d0-9f3d-42a9-9f4e-16f9e24cba19", + "id": "6db67bc1-7fff-47e7-a931-f797b1f76732", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "6db67bc1-7fff-47e7-a931-f797b1f76732", + "gamma": 0.3, + "id": "92bc1447-2b30-498c-ae8a-c67904fc82b2", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "Network - Event Throughput — ArcSight [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d618f6ab-e55c-42e9-b71e-f65e45630795", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d96eec3e-0f71-4949-81bb-216a5e9a6d08.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d96eec3e-0f71-4949-81bb-216a5e9a6d08.json new file mode 100644 index 00000000000..6bc92c9ff6c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-d96eec3e-0f71-4949-81bb-216a5e9a6d08.json @@ -0,0 +1,135 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Device Metrics Overview [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Event Count" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Sources", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Destinations", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "12", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Device Metrics Overview [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-d96eec3e-0f71-4949-81bb-216a5e9a6d08", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-dc5d284d-a136-4207-9557-657f0972d534.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-dc5d284d-a136-4207-9557-657f0972d534.json new file mode 100644 index 00000000000..80a694b969b --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-dc5d284d-a136-4207-9557-657f0972d534.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 20 Behaviors by Outcome — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Behavior", + "field": "cef.extensions.categoryBehavior", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 3 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "right", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 20 Behaviors by Outcome — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-dc5d284d-a136-4207-9557-657f0972d534", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e345045a-26d7-44bf-8829-0d48d78419b5.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e345045a-26d7-44bf-8829-0d48d78419b5.json new file mode 100644 index 00000000000..1d6cbbeea3a --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e345045a-26d7-44bf-8829-0d48d78419b5.json @@ -0,0 +1,108 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Countries by Event [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total Events" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 35 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Source Countries by Event [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e345045a-26d7-44bf-8829-0d48d78419b5", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e386b2cc-4e81-4e16-bbfa-65cf583dd247.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e386b2cc-4e81-4e16-bbfa-65cf583dd247.json new file mode 100644 index 00000000000..10fc9605660 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e386b2cc-4e81-4e16-bbfa-65cf583dd247.json @@ -0,0 +1,81 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Sources by Destinations — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Host", + "field": "source.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Destination Host", + "field": "destination.domain", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addTooltip": true, + "distinctColors": true, + "isDonut": true, + "legendDisplay": "show", + "legendPosition": "bottom", + "legendSize": "auto", + "palette": { + "name": "kibana_palette", + "type": "palette" + } + }, + "title": "Top 10 Sources by Destinations — ArcSight [Filebeat CEF]", + "type": "pie" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e386b2cc-4e81-4e16-bbfa-65cf583dd247", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e3ca7b8d-7c49-451a-a51e-6696ca9b223d.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e3ca7b8d-7c49-451a-a51e-6696ca9b223d.json new file mode 100644 index 00000000000..04af2058bcc --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e3ca7b8d-7c49-451a-a51e-6696ca9b223d.json @@ -0,0 +1,168 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Source [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "background_color_rules": [ + { + "id": "2fddda5e-d6fc-4581-bbb7-574e1017ae8f" + } + ], + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceType:\"Firewall\" OR cef.extensions.categoryDeviceGroup:\"/IDS/Network\" OR cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "gauge_color_rules": [ + { + "id": "3ed9a6b9-fd2e-4e0d-bd83-7ad467b3c8a4" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": "0", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "formatter": "number", + "id": "04c44192-1112-4515-a8d9-e9e13215aecf", + "label": "Events", + "line_width": "3", + "metrics": [ + { + "id": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "c5dbb050-fc10-4a0d-abe0-bc093db6cf0e", + "gamma": 0.3, + "id": "e5a48d9d-7834-4da7-8d78-7d4528136b9b", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "sigma": "", + "type": "moving_average", + "window": "10" + } + ], + "point_size": "0", + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(244,78,59,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/Firewall\"" + }, + "id": "0c929603-fc92-4ebc-a963-fe2795417d89", + "label": "Firewall Events" + }, + { + "color": "rgba(254,146,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/IDS/Network\"" + }, + "id": "7798827b-87ab-436b-9e62-9fe36143eb9b", + "label": "Intrusion Detection Events" + }, + { + "color": "rgba(252,220,0,1)", + "filter": { + "language": "lucene", + "query": "cef.extensions.categoryDeviceGroup:\"/VPN\"" + }, + "id": "490f7ad7-8218-45f9-85a9-a4dd9ed7da13", + "label": "VPN" + } + ], + "split_mode": "filters", + "stacked": "none", + "steps": 0, + "terms_field": "observer.hostname", + "terms_order_by": null + }, + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(0,156,224,1)", + "fill": "0.5", + "formatter": "number", + "id": "29d6131a-5143-4a64-b597-9538692f0269", + "label": "Moving Average by Device Hosts", + "line_width": 1, + "metrics": [ + { + "id": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "type": "count" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "dc74afdf-64ad-47d6-bbed-114e09d12255", + "gamma": 0.3, + "id": "87e21aaa-12eb-4213-bb37-41cb19219240", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 1, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "observer.hostname", + "terms_size": "10" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Source [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e3ca7b8d-7c49-451a-a51e-6696ca9b223d", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e5126c06-0bbd-40d8-922f-3ece07b4eadf.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e5126c06-0bbd-40d8-922f-3ece07b4eadf.json new file mode 100644 index 00000000000..c9de8cab9e9 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e5126c06-0bbd-40d8-922f-3ece07b4eadf.json @@ -0,0 +1,117 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Bandwidth Utilization [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "background_color": null, + "bar_color_rules": [ + { + "id": "23db5bf6-f787-474e-86ab-76362432e984" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "kuery", + "query": "" + }, + "id": "ec53a1d3-213c-4b0f-a074-5005a84cdb83", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "d27f09dc-b07e-493f-a223-a85033ad6548", + "label": "Inbound", + "line_width": 1, + "metrics": [ + { + "field": "source.bytes", + "id": "9ce9ec3a-2f11-4935-91b2-531494d2a619", + "type": "sum" + } + ], + "override_index_pattern": 1, + "point_size": 1, + "seperate_axis": 0, + "series_drop_last_bucket": 1, + "series_index_pattern": "filebeat-*", + "series_time_field": "@timestamp", + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "terms_field": "observer.hostname", + "terms_order_by": "_count" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(244,78,59,1)", + "fill": 0.5, + "formatter": "bytes", + "id": "b1ef2c75-5916-469d-8790-5b213367a5a0", + "label": "Outbound", + "line_width": 1, + "metrics": [ + { + "field": "destination.bytes", + "id": "11b1852f-9b62-4e96-8128-522e6c5bf16d", + "type": "sum" + }, + { + "id": "2a6b00bf-1658-4d02-b4e2-61ad6e4c3a9b", + "script": "params.outbound \u003e 0 ? params.outbound * -1 : 0", + "type": "calculation", + "variables": [ + { + "field": "11b1852f-9b62-4e96-8128-522e6c5bf16d", + "id": "c57067f2-2927-41d8-97f4-9f47b3b3bcae", + "name": "outbound" + } + ] + } + ], + "override_index_pattern": 1, + "point_size": 1, + "seperate_axis": 0, + "series_drop_last_bucket": 1, + "series_index_pattern": "filebeat-*", + "series_time_field": "@timestamp", + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "steps": 0 + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Bandwidth Utilization [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e5126c06-0bbd-40d8-922f-3ece07b4eadf", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e6d29418-b100-44f6-a0bb-059050e973ae.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e6d29418-b100-44f6-a0bb-059050e973ae.json new file mode 100644 index 00000000000..673fe8444fe --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e6d29418-b100-44f6-a0bb-059050e973ae.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Users [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Users", + "field": "source.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Source Users [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e6d29418-b100-44f6-a0bb-059050e973ae", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NTksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e98f402f-d0ee-412a-995f-f3c455cac1ec.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e98f402f-d0ee-412a-995f-f3c455cac1ec.json new file mode 100644 index 00000000000..3d2b591439e --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e98f402f-d0ee-412a-995f-f3c455cac1ec.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Users [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Users", + "field": "destination.user.name", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 60, + "minFontSize": 10, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Users [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e98f402f-d0ee-412a-995f-f3c455cac1ec", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e9d591ca-fc0c-40fd-b3b6-391df6ace611.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e9d591ca-fc0c-40fd-b3b6-391df6ace611.json new file mode 100644 index 00000000000..44b544b668d --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-e9d591ca-fc0c-40fd-b3b6-391df6ace611.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Destination Addresses [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 10 Destination Addresses [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-e9d591ca-fc0c-40fd-b3b6-391df6ace611", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-d687d631-96df-4c49-99ee-9e51c7053084", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjEsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-ea681829-8689-4f48-8930-76e6a78ac8ad.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-ea681829-8689-4f48-8930-76e6a78ac8ad.json new file mode 100644 index 00000000000..edc384ae09e --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-ea681829-8689-4f48-8930-76e6a78ac8ad.json @@ -0,0 +1,114 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Behaviors by Outcome [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 9,000": "rgb(255,255,204)", + "18,000 - 27,000": "rgb(254,225,135)", + "27,000 - 36,000": "rgb(254,201,101)", + "36,000 - 45,000": "rgb(254,171,73)", + "45,000 - 54,000": "rgb(253,141,60)", + "54,000 - 63,000": "rgb(252,91,46)", + "63,000 - 72,000": "rgb(237,47,34)", + "72,000 - 81,000": "rgb(212,16,32)", + "81,000 - 90,000": "rgb(176,0,38)", + "9,000 - 18,000": "rgb(255,241,170)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Event Type", + "field": "event.action", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "event.outcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Yellow to Red", + "colorsNumber": 10, + "colorsRange": [], + "enableHover": true, + "invertColors": false, + "legendPosition": "right", + "legendSize": "auto", + "percentageMode": false, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "Top 10 Behaviors by Outcome [Filebeat CEF]", + "type": "heatmap" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-ea681829-8689-4f48-8930-76e6a78ac8ad", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-12352cbd-ccbb-4ede-a98b-c4c82a358516", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjIsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-eb86beed-6c68-4872-bb3e-e1df70f2c34a.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-eb86beed-6c68-4872-bb3e-e1df70f2c34a.json new file mode 100644 index 00000000000..d2d5d12bae1 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-eb86beed-6c68-4872-bb3e-e1df70f2c34a.json @@ -0,0 +1,153 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Outcome by Device Type — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "colors": { + "/Failure": "#BF1B00", + "/Success": "#629E51" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Firewall Types", + "field": "cef.extensions.categoryDeviceType", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Event Outcome", + "field": "cef.extensions.categoryOutcome", + "order": "desc", + "orderBy": "1", + "size": 3 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "rotate": 75, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Firewall Types" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "orderBucketsBySum": true, + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "percentage", + "type": "square root" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Outcome by Device Type — ArcSight [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-eb86beed-6c68-4872-bb3e-e1df70f2c34a", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjMsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f1582760-9a38-44f8-bc1f-9832b967a034.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f1582760-9a38-44f8-bc1f-9832b967a034.json new file mode 100644 index 00000000000..93b0d90b072 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f1582760-9a38-44f8-bc1f-9832b967a034.json @@ -0,0 +1,144 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Destination Ports by Outcomes [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "destination.port", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "event.outcome", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "group", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "destination.port: Descending" + }, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "interpolate": "linear", + "legendPosition": "right", + "legendSize": "auto", + "radiusRatio": 9, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "showCircles": true, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "defaultYExtents": true, + "mode": "normal", + "setYExtents": false, + "type": "square root" + }, + "show": true, + "style": {}, + "title": {}, + "type": "value" + } + ] + }, + "title": "Destination Ports by Outcomes [Filebeat CEF]", + "type": "histogram" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-f1582760-9a38-44f8-bc1f-9832b967a034", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-8eaeaa88-771f-4721-8f35-f94da4ea6009", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjQsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f1da0e00-c6a0-4e86-b9df-403f81818f62.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f1da0e00-c6a0-4e86-b9df-403f81818f62.json new file mode 100644 index 00000000000..958dc208917 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f1da0e00-c6a0-4e86-b9df-403f81818f62.json @@ -0,0 +1,109 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Source Countries by Events — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total Events" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customLabel": "Source Country", + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 10 + }, + "schema": "bucket", + "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Source Addresses", + "field": "source.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Destination Addresses", + "field": "destination.ip" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "5", + "params": { + "customLabel": "Destination Ports", + "field": "destination.port" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "perPage": 10, + "showMeticsAtAllLevels": false, + "showPartialRows": false, + "showToolbar": true, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "Top 10 Source Countries by Events — ArcSight [Filebeat CEF]", + "type": "table" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-f1da0e00-c6a0-4e86-b9df-403f81818f62", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjUsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f3b8b32b-e4a0-4738-b9d1-ca878f3530b1.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f3b8b32b-e4a0-4738-b9d1-ca878f3530b1.json new file mode 100644 index 00000000000..c2870c16de0 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f3b8b32b-e4a0-4738-b9d1-ca878f3530b1.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 5 Source Countries — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.geo.country_iso_code", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 18, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "linear" + }, + "title": "Top 5 Source Countries — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-f3b8b32b-e4a0-4738-b9d1-ca878f3530b1", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-2f6c8e48-8c6a-476d-82e8-47c723c39715", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjYsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f5fde1d9-50ff-4a55-aa6a-724a8d6c05b2.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f5fde1d9-50ff-4a55-aa6a-724a8d6c05b2.json new file mode 100644 index 00000000000..d8a9b47206b --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-f5fde1d9-50ff-4a55-aa6a-724a8d6c05b2.json @@ -0,0 +1,125 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Endpoint OS Metrics Overview — ArcSight [Filebeat CEF]", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 100": "rgb(0,104,55)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total Events" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "6", + "params": { + "customLabel": "Devices", + "field": "observer.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "7", + "params": { + "customLabel": "Event Types", + "field": "cef.extensions.categoryBehavior" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "8", + "params": { + "customLabel": "Event Outcomes", + "field": "cef.extensions.categoryOutcome" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "listeners": {}, + "params": { + "addLegend": false, + "addTooltip": true, + "fontSize": "30", + "gauge": { + "autoExtend": false, + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 100 + } + ], + "gaugeColorMode": "None", + "gaugeStyle": "Full", + "gaugeType": "Metric", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "#333", + "labels": false, + "show": false, + "width": 2 + }, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": "20", + "labelColor": false, + "subText": "" + }, + "type": "simple", + "useRange": false, + "verticalSplit": false + }, + "handleNoResults": true, + "type": "gauge" + }, + "title": "Endpoint OS Metrics Overview — ArcSight [Filebeat CEF]", + "type": "metric" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-f5fde1d9-50ff-4a55-aa6a-724a8d6c05b2", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-69d6e511-7744-429a-9aa4-ceae2222db94", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjcsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fb21024b-49db-405d-85fa-ab7fd43e3753.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fb21024b-49db-405d-85fa-ab7fd43e3753.json new file mode 100644 index 00000000000..f387ae31a33 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fb21024b-49db-405d-85fa-ab7fd43e3753.json @@ -0,0 +1,124 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Events by Direction [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "cef.device.product:\"DNS Trace Log\"" + }, + "id": "be556a57-cd1c-496c-8714-0bd210947c85", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "bar", + "color": "#68BC00", + "fill": "0.2", + "filter": { + "language": "lucene", + "query": "device" + }, + "formatter": "number", + "id": "9aae7344-9de9-4378-b21d-296cb964f93b", + "label": "Inbound Requests", + "line_width": 1, + "metrics": [ + { + "id": "1cd0b964-45cf-408e-a7e4-e26955f8a3b0", + "type": "count" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(0,156,224,1)", + "filter": { + "language": "lucene", + "query": "deviceDirection:\"0\"" + }, + "id": "f860f6e0-fbd4-4949-8046-6300322dfe84", + "label": "Inbound Requests" + } + ], + "split_mode": "filters", + "stacked": "none" + }, + { + "axis_position": "right", + "chart_type": "bar", + "color": "#68BC00", + "fill": "0.2", + "formatter": "number", + "id": "ed1abe18-e01b-4202-9db4-06fda10692e0", + "label": "Outbound Requests", + "line_width": 1, + "metrics": [ + { + "id": "cfbcfc79-394b-4ec0-a2c2-7a47177d6469", + "type": "count" + }, + { + "id": "6bc37118-ddac-41ec-85b3-9db7e1b3636b", + "script": "params.outbound \u003e 0 ? params.outbound * -1 : 0", + "type": "calculation", + "variables": [ + { + "field": "cfbcfc79-394b-4ec0-a2c2-7a47177d6469", + "id": "f73f4f22-03d5-446a-b031-04eee531e3cc", + "name": "outbound" + } + ] + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_filters": [ + { + "color": "rgba(211,49,21,1)", + "filter": { + "language": "lucene", + "query": "deviceDirection:\"1\"" + }, + "id": "a9c50e1b-8f11-4bc2-9077-bb8870ed0b62", + "label": "Outbound Requests" + } + ], + "split_mode": "filters", + "stacked": "none" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries", + "use_kibana_indexes": false + }, + "title": "Events by Direction [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-fb21024b-49db-405d-85fa-ab7fd43e3753", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjgsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fbaeed43-bca4-4206-9a42-935140b2b48e.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fbaeed43-bca4-4206-9a42-935140b2b48e.json new file mode 100644 index 00000000000..b88f1810f91 --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fbaeed43-bca4-4206-9a42-935140b2b48e.json @@ -0,0 +1,103 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Endpoint Average EPS [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "listeners": {}, + "params": { + "axis_formatter": "number", + "axis_position": "left", + "bar_color_rules": [ + { + "id": "85a1c642-9781-430d-b84b-b28cb2a42fb4" + } + ], + "drop_last_bucket": 1, + "filter": { + "language": "lucene", + "query": "event.dataset:\"cef.log\"" + }, + "gauge_color_rules": [ + { + "id": "03a2fd72-fc9c-4582-9133-20af36217180" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "hide_last_value_indicator": true, + "id": "b7a85957-123e-4e25-9e8e-ff7992c9b2b9", + "index_pattern": "filebeat-*", + "interval": "auto", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(0,156,224,1)", + "fill": 0.5, + "formatter": "number", + "id": "b4373ffd-9660-4206-afd6-d4867ac7dbdf", + "label": "Event Throughput", + "line_width": 1, + "metrics": [ + { + "id": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "type": "count" + }, + { + "field": "b1a48389-d799-4eba-8b98-7ee8ef0bb440", + "id": "7c5c44cc-17bd-4206-a100-b8996cd3d11a", + "type": "cumulative_sum" + }, + { + "field": "7c5c44cc-17bd-4206-a100-b8996cd3d11a", + "id": "215c5225-5368-40e6-8fcd-2b0026babba0", + "type": "derivative", + "unit": "1s" + }, + { + "alpha": 0.3, + "beta": 0.1, + "field": "215c5225-5368-40e6-8fcd-2b0026babba0", + "gamma": 0.3, + "id": "f4dfe09a-e397-4287-ab99-3206516cded3", + "model_type": "simple", + "multiplicative": false, + "period": 1, + "type": "moving_average", + "window": "10" + } + ], + "point_size": 1, + "seperate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}} / s" + } + ], + "show_legend": 1, + "time_field": "@timestamp", + "type": "gauge", + "use_kibana_indexes": false + }, + "title": "Endpoint Average EPS [Filebeat CEF]", + "type": "metrics" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-fbaeed43-bca4-4206-9a42-935140b2b48e", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NjksMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fe42b0d8-1fd3-4d8a-a4ff-4ca9915d2644.json b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fe42b0d8-1fd3-4d8a-a4ff-4ca9915d2644.json new file mode 100644 index 00000000000..88596e91b3c --- /dev/null +++ b/x-pack/filebeat/module/cef/_meta/kibana/8/visualization/cef-fe42b0d8-1fd3-4d8a-a4ff-4ca9915d2644.json @@ -0,0 +1,65 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchRefName": "search_0", + "title": "Top 10 Application Protocols — ArcSight [Filebeat CEF]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "network.application", + "order": "desc", + "orderBy": "1", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "maxFontSize": 72, + "minFontSize": 26, + "orientation": "single", + "palette": { + "name": "kibana_palette", + "type": "palette" + }, + "scale": "square root" + }, + "title": "Top 10 Application Protocols — ArcSight [Filebeat CEF]", + "type": "tagcloud" + } + }, + "coreMigrationVersion": "8.3.3", + "id": "cef-fe42b0d8-1fd3-4d8a-a4ff-4ca9915d2644", + "migrationVersion": { + "visualization": "8.3.0" + }, + "references": [ + { + "id": "cef-1da6dac6-7c41-47c5-bcba-567b5e83c7e1", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2022-08-24T00:29:51.550Z", + "version": "WzI0NzAsMV0=" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log index 42ffa8a85d7..7a90091d6fc 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log @@ -91,3 +91,12 @@ May 5 19:02:25 dev01: %ASA-4-733100: [ Scanning] drop rate-2 exceeded. Current May 5 19:02:25 dev01: %ASA-4-733100: [ 192.168.0.1] drop rate-1 exceeded. Current burst rate is 0 per second, max configured rate is 10; Current average rate is 5 per second, max configured rate is 5; Cumulative total count is 6018 May 5 19:02:25 dev01: %ASA-4-733100: [ Port-5432 5432] drop rate-1 exceeded. Current burst rate is 8 per second, max configured rate is 10; Current average rate is 20 per second, max configured rate is 5; Cumulative total count is 12466 May 5 19:02:25 dev01: %ASA-4-733100: [ RDP 3389] drop rate-1 exceeded. Current burst rate is 63 per second, max configured rate is 10; Current average rate is 5 per second, max configured rate is 5; Cumulative total count is 3054 +May 5 19:02:25 dev01: %ASA-6-113004: AAA user authentication Successful: server = 81.2.69.144 , User = alice +May 5 19:02:25 dev01: %ASA-6-113004: AAA user authorization Successful: server = 81.2.69.144 , User = alice +May 5 19:02:25 dev01: %ASA-6-113005: AAA user authentication Rejected: reason = AAA failure: server = 81.2.69.144 : user = *****: user IP = 172.31.98.44 +May 5 19:02:25 dev01: %ASA-6-113012: AAA user authentication Successful: local database: user = alice +May 5 19:02:25 dev01: %ASA-3-113021: Attempted console login failed. User eve did NOT have appropriate Admin Rights. +May 5 19:02:25 dev01: %ASA-6-716039: Authentication: rejected, group = malcorp user = eve , Session Type: admin +May 5 19:02:25 dev01: %ASA-6-716039: Authentication: rejected, group = malcorp user = malory , Session Type: WebVPN +May 5 19:02:25 dev01: %ASA-6-716039: Group User IP <172.31.98.44> Authentication: rejected, Session Type: Admin. +May 5 19:02:25 dev01: %ASA-6-716039: Group User IP <172.31.98.44> Authentication: rejected, Session Type: WebVPN. diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index c6feef5d7e8..34dfa5f225b 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -4593,5 +4593,243 @@ "cisco-asa", "forwarded" ] + }, + { + "cisco.asa.message_id": "113004", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113004, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-113004: AAA user authentication Successful: server = 81.2.69.144 , User = alice", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "informational", + "log.offset": 13920, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "113004", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113004, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-113004: AAA user authorization Successful: server = 81.2.69.144 , User = alice", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "informational", + "log.offset": 14030, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "113005", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113005, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-113005: AAA user authentication Rejected: reason = AAA failure: server = 81.2.69.144 : user = *****: user IP = 172.31.98.44", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "informational", + "log.offset": 14139, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "113012", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113012, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-113012: AAA user authentication Successful: local database: user = alice", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "informational", + "log.offset": 14293, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "113021", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113021, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-3-113021: Attempted console login failed. User eve did NOT have appropriate Admin Rights.", + "event.severity": 3, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "error", + "log.offset": 14396, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "716039", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 716039, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-716039: Authentication: rejected, group = malcorp user = eve , Session Type: admin", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "informational", + "log.offset": 14514, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "716039", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 716039, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-716039: Authentication: rejected, group = malcorp user = malory , Session Type: WebVPN", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "dev01", + "input.type": "log", + "log.level": "informational", + "log.offset": 14627, + "observer.hostname": "dev01", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "dev01" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/asa/test/non-canonical.log b/x-pack/filebeat/module/cisco/asa/test/non-canonical.log new file mode 100644 index 00000000000..81cc35122d9 --- /dev/null +++ b/x-pack/filebeat/module/cisco/asa/test/non-canonical.log @@ -0,0 +1,19 @@ +Jul 15 13:38:14 216.160.83.56 : %ASA-6-302013: Built inbound TCP connection 3263493120 for DMZ:shule/5802 (shule/5802) to SERVERS:10.10.227.121/80 (10.10.227.121/80) +Jul 15 13:38:11 216.160.83.56 : %ASA-6-302013: Built outbound TCP connection 3263492189 for MG:exp_srv/10050 (exp_srv/10050) to SERVERS:10.10.227.170/46145 (10.10.224.1/46145) +Jul 15 13:38:08 81.2.69.142 %ASA-6-302015: Built outbound UDP connection 743108828 for outside:ns10/53 (ns10/53) to MND_sec:192.168.174.100/48347 (89.160.20.128/48347) +Jul 15 13:38:03 81.2.69.142 %ASA-6-302015: Built outbound UDP connection 743108738 for outside:ns10/53 (ns10/53) to MND_sec:192.168.174.100/55653 (81.2.69.192/55653) +Jul 15 13:36:59 216.160.83.56 : %ASA-6-106015: Deny TCP (no connection) from 10.12.227.40/389 to exp-angle/54703 flags RST on interface SH_INFRA_MGT +Jul 15 13:36:39 216.160.83.56 : %ASA-6-106015: Deny TCP (no connection) from 89.160.20.128/56594 to sh-mailgw1/25 flags FIN ACK on interface outside +Jul 15 13:38:47 216.160.83.56 : %ASA-6-305012: Teardown dynamic UDP translation from SERVERS:exp-wait/62409 to outside:81.2.69.142/62409 duration 0:00:41 +Jul 15 13:37:33 216.160.83.56 : %ASA-6-305012: Teardown dynamic UDP translation from SERVERS:exp-wait/56421 to outside:81.2.69.142/56421 duration 0:00:30 +Jul 15 13:39:04 216.160.83.56 : %ASA-6-305011: Built dynamic TCP translation from SERVERS:exp-srv/50578 to outside:81.2.69.142/50578 +Jul 15 13:37:02 216.160.83.56 : %ASA-6-305011: Built dynamic UDP translation from SERVERS:exp-wait/56570 to outside:81.2.69.142/56570 +Jul 15 13:18:06 216.160.83.56 : %ASA-4-106023: Deny tcp src MG:exp_srv/64593 dst SH_OSS:89.160.20.128/2511 by access-group "MGT_access_in" [0x0, 0x0] +Jul 15 01:18:01 216.160.83.56 : %ASA-4-106023: Deny tcp src MG:exp_srv/63513 dst SH_OSS:89.160.20.128/2511 by access-group "MGT_access_in" [0x0, 0x0] +Jul 15 13:30:09 81.2.69.142 %ASA-6-302020: Built inbound ICMP connection for faddr eth0_fw/6553 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0 +Jul 14 01:45:09 81.2.69.142 %ASA-6-302020: Built inbound ICMP connection for faddr eth0_fw/8396 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0 +Jul 15 13:30:09 81.2.69.142 %ASA-6-302021: Teardown ICMP connection for faddr eth0_fw/6553 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0 +Jul 14 01:45:09 81.2.69.142 %ASA-6-302021: Teardown ICMP connection for faddr eth0_fw/8396 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0 +Jul 15 12:18:51 81.2.69.192 %ASA-6-113039: Group User IP <216.160.83.56> AnyConnect parent session started. +Jul 1 09:27:13 216.160.83.56 : %ASA-6-113039: Group User IP <81.2.69.192> AnyConnect parent session started. +Jun 14 01:22:47 81.2.69.142 %ASA-5-304001: 192.168.14.22 Accessed URL mirror:http://mirror.example.com/path/to/resource diff --git a/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json new file mode 100644 index 00000000000..7a78a5a2bf4 --- /dev/null +++ b/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json @@ -0,0 +1,1044 @@ +[ + { + "cisco.asa.connection_id": "3263493120", + "cisco.asa.destination_interface": "SERVERS", + "cisco.asa.mapped_destination_ip": "10.10.227.121", + "cisco.asa.mapped_destination_port": 80, + "cisco.asa.mapped_source_host": "shule", + "cisco.asa.mapped_source_port": 5802, + "cisco.asa.message_id": "302013", + "cisco.asa.source_interface": "DMZ", + "destination.address": "10.10.227.121", + "destination.ip": "10.10.227.121", + "destination.port": 80, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 302013, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302013: Built inbound TCP connection 3263493120 for DMZ:shule/5802 (shule/5802) to SERVERS:10.10.227.121/80 (10.10.227.121/80)", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 0, + "network.direction": "inbound", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "SERVERS", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "DMZ", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "shule" + ], + "related.ip": [ + "10.10.227.121" + ], + "service.type": "cisco", + "source.address": "shule", + "source.domain": "shule", + "source.port": 5802, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.connection_id": "3263492189", + "cisco.asa.destination_interface": "SERVERS", + "cisco.asa.mapped_destination_ip": "10.10.224.1", + "cisco.asa.mapped_destination_port": 46145, + "cisco.asa.mapped_source_host": "exp_srv", + "cisco.asa.mapped_source_port": 10050, + "cisco.asa.message_id": "302013", + "cisco.asa.source_interface": "MG", + "destination.address": "10.10.227.170", + "destination.ip": "10.10.227.170", + "destination.nat.ip": "10.10.224.1", + "destination.port": 46145, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 302013, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302013: Built outbound TCP connection 3263492189 for MG:exp_srv/10050 (exp_srv/10050) to SERVERS:10.10.227.170/46145 (10.10.224.1/46145)", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 166, + "network.direction": "outbound", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "SERVERS", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "MG", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp_srv" + ], + "related.ip": [ + "10.10.224.1", + "10.10.227.170" + ], + "service.type": "cisco", + "source.address": "exp_srv", + "source.domain": "exp_srv", + "source.port": 10050, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.connection_id": "743108828", + "cisco.asa.destination_interface": "MND_sec", + "cisco.asa.mapped_destination_ip": "89.160.20.128", + "cisco.asa.mapped_destination_port": 48347, + "cisco.asa.mapped_source_host": "ns10", + "cisco.asa.mapped_source_port": 53, + "cisco.asa.message_id": "302015", + "cisco.asa.source_interface": "outside", + "destination.address": "192.168.174.100", + "destination.ip": "192.168.174.100", + "destination.nat.ip": "89.160.20.128", + "destination.port": 48347, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 302015, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302015: Built outbound UDP connection 743108828 for outside:ns10/53 (ns10/53) to MND_sec:192.168.174.100/48347 (89.160.20.128/48347)", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "informational", + "log.offset": 342, + "network.direction": "outbound", + "network.iana_number": 17, + "network.transport": "udp", + "observer.egress.interface.name": "MND_sec", + "observer.hostname": "81.2.69.142", + "observer.ingress.interface.name": "outside", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "ns10" + ], + "related.ip": [ + "192.168.174.100", + "89.160.20.128" + ], + "service.type": "cisco", + "source.address": "ns10", + "source.domain": "ns10", + "source.port": 53, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.connection_id": "743108738", + "cisco.asa.destination_interface": "MND_sec", + "cisco.asa.mapped_destination_ip": "81.2.69.192", + "cisco.asa.mapped_destination_port": 55653, + "cisco.asa.mapped_source_host": "ns10", + "cisco.asa.mapped_source_port": 53, + "cisco.asa.message_id": "302015", + "cisco.asa.source_interface": "outside", + "destination.address": "192.168.174.100", + "destination.ip": "192.168.174.100", + "destination.nat.ip": "81.2.69.192", + "destination.port": 55653, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 302015, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302015: Built outbound UDP connection 743108738 for outside:ns10/53 (ns10/53) to MND_sec:192.168.174.100/55653 (81.2.69.192/55653)", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "informational", + "log.offset": 510, + "network.direction": "outbound", + "network.iana_number": 17, + "network.transport": "udp", + "observer.egress.interface.name": "MND_sec", + "observer.hostname": "81.2.69.142", + "observer.ingress.interface.name": "outside", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "ns10" + ], + "related.ip": [ + "192.168.174.100", + "81.2.69.192" + ], + "service.type": "cisco", + "source.address": "ns10", + "source.domain": "ns10", + "source.port": 53, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "106015", + "cisco.asa.source_interface": "SH_INFRA_MGT", + "destination.address": "exp-angle", + "destination.domain": "exp-angle", + "destination.port": 54703, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 106015, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-106015: Deny TCP (no connection) from 10.12.227.40/389 to exp-angle/54703 flags RST on interface SH_INFRA_MGT", + "event.outcome": "success", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "denied" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 676, + "network.iana_number": 6, + "network.transport": "tcp", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "SH_INFRA_MGT", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp-angle" + ], + "related.ip": [ + "10.12.227.40" + ], + "service.type": "cisco", + "source.address": "10.12.227.40", + "source.ip": "10.12.227.40", + "source.port": 389, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "106015", + "cisco.asa.source_interface": "outside", + "destination.address": "sh-mailgw1", + "destination.domain": "sh-mailgw1", + "destination.port": 25, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 106015, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-106015: Deny TCP (no connection) from 89.160.20.128/56594 to sh-mailgw1/25 flags FIN ACK on interface outside", + "event.outcome": "success", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "denied" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 825, + "network.iana_number": 6, + "network.transport": "tcp", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "outside", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "sh-mailgw1" + ], + "related.ip": [ + "89.160.20.128" + ], + "service.type": "cisco", + "source.address": "89.160.20.128", + "source.as.number": 29518, + "source.as.organization.name": "Bredband2 AB", + "source.geo.city_name": "Link\u00f6ping", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "SE", + "source.geo.country_name": "Sweden", + "source.geo.location.lat": 58.4167, + "source.geo.location.lon": 15.6167, + "source.geo.region_iso_code": "SE-E", + "source.geo.region_name": "\u00d6sterg\u00f6tland County", + "source.ip": "89.160.20.128", + "source.port": 56594, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.destination_interface": "outside", + "cisco.asa.message_id": "305012", + "cisco.asa.source_interface": "SERVERS", + "destination.address": "81.2.69.142", + "destination.geo.city_name": "London", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "GB", + "destination.geo.country_name": "United Kingdom", + "destination.geo.location.lat": 51.5142, + "destination.geo.location.lon": -0.0931, + "destination.geo.region_iso_code": "GB-ENG", + "destination.geo.region_name": "England", + "destination.ip": "81.2.69.142", + "destination.port": 62409, + "event.action": "flow-expiration", + "event.category": [ + "network" + ], + "event.code": 305012, + "event.dataset": "cisco.asa", + "event.duration": 41000000000, + "event.end": "2022-07-15T13:38:47.000-02:00", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-305012: Teardown dynamic UDP translation from SERVERS:exp-wait/62409 to outside:81.2.69.142/62409 duration 0:00:41", + "event.severity": 6, + "event.start": "2022-07-15T15:38:06.000Z", + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 974, + "network.iana_number": 17, + "network.transport": "udp", + "observer.egress.interface.name": "outside", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "SERVERS", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp-wait" + ], + "related.ip": [ + "81.2.69.142" + ], + "service.type": "cisco", + "source.address": "exp-wait", + "source.domain": "exp-wait", + "source.port": 62409, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.destination_interface": "outside", + "cisco.asa.message_id": "305012", + "cisco.asa.source_interface": "SERVERS", + "destination.address": "81.2.69.142", + "destination.geo.city_name": "London", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "GB", + "destination.geo.country_name": "United Kingdom", + "destination.geo.location.lat": 51.5142, + "destination.geo.location.lon": -0.0931, + "destination.geo.region_iso_code": "GB-ENG", + "destination.geo.region_name": "England", + "destination.ip": "81.2.69.142", + "destination.port": 56421, + "event.action": "flow-expiration", + "event.category": [ + "network" + ], + "event.code": 305012, + "event.dataset": "cisco.asa", + "event.duration": 30000000000, + "event.end": "2022-07-15T13:37:33.000-02:00", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-305012: Teardown dynamic UDP translation from SERVERS:exp-wait/56421 to outside:81.2.69.142/56421 duration 0:00:30", + "event.severity": 6, + "event.start": "2022-07-15T15:37:03.000Z", + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 1128, + "network.iana_number": 17, + "network.transport": "udp", + "observer.egress.interface.name": "outside", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "SERVERS", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp-wait" + ], + "related.ip": [ + "81.2.69.142" + ], + "service.type": "cisco", + "source.address": "exp-wait", + "source.domain": "exp-wait", + "source.port": 56421, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.destination_interface": "outside", + "cisco.asa.message_id": "305011", + "cisco.asa.source_interface": "SERVERS", + "destination.address": "81.2.69.142", + "destination.geo.city_name": "London", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "GB", + "destination.geo.country_name": "United Kingdom", + "destination.geo.location.lat": 51.5142, + "destination.geo.location.lon": -0.0931, + "destination.geo.region_iso_code": "GB-ENG", + "destination.geo.region_name": "England", + "destination.ip": "81.2.69.142", + "destination.port": 50578, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 305011, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-305011: Built dynamic TCP translation from SERVERS:exp-srv/50578 to outside:81.2.69.142/50578", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 1282, + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "outside", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "SERVERS", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp-srv" + ], + "related.ip": [ + "81.2.69.142" + ], + "service.type": "cisco", + "source.address": "exp-srv", + "source.domain": "exp-srv", + "source.port": 50578, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.destination_interface": "outside", + "cisco.asa.message_id": "305011", + "cisco.asa.source_interface": "SERVERS", + "destination.address": "81.2.69.142", + "destination.geo.city_name": "London", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "GB", + "destination.geo.country_name": "United Kingdom", + "destination.geo.location.lat": 51.5142, + "destination.geo.location.lon": -0.0931, + "destination.geo.region_iso_code": "GB-ENG", + "destination.geo.region_name": "England", + "destination.ip": "81.2.69.142", + "destination.port": 56570, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 305011, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-305011: Built dynamic UDP translation from SERVERS:exp-wait/56570 to outside:81.2.69.142/56570", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 1415, + "network.iana_number": 17, + "network.transport": "udp", + "observer.egress.interface.name": "outside", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "SERVERS", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp-wait" + ], + "related.ip": [ + "81.2.69.142" + ], + "service.type": "cisco", + "source.address": "exp-wait", + "source.domain": "exp-wait", + "source.port": 56570, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.destination_interface": "SH_OSS", + "cisco.asa.message_id": "106023", + "cisco.asa.rule_name": "MGT_access_in", + "cisco.asa.source_interface": "MG", + "destination.address": "89.160.20.128", + "destination.as.number": 29518, + "destination.as.organization.name": "Bredband2 AB", + "destination.geo.city_name": "Link\u00f6ping", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "SE", + "destination.geo.country_name": "Sweden", + "destination.geo.location.lat": 58.4167, + "destination.geo.location.lon": 15.6167, + "destination.geo.region_iso_code": "SE-E", + "destination.geo.region_name": "\u00d6sterg\u00f6tland County", + "destination.ip": "89.160.20.128", + "destination.port": 2511, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 106023, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-4-106023: Deny tcp src MG:exp_srv/64593 dst SH_OSS:89.160.20.128/2511 by access-group \"MGT_access_in\" [0x0, 0x0]", + "event.outcome": "success", + "event.severity": 4, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "denied" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "warning", + "log.offset": 1549, + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "SH_OSS", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "MG", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp_srv" + ], + "related.ip": [ + "89.160.20.128" + ], + "service.type": "cisco", + "source.address": "exp_srv", + "source.domain": "exp_srv", + "source.port": 64593, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.destination_interface": "SH_OSS", + "cisco.asa.message_id": "106023", + "cisco.asa.rule_name": "MGT_access_in", + "cisco.asa.source_interface": "MG", + "destination.address": "89.160.20.128", + "destination.as.number": 29518, + "destination.as.organization.name": "Bredband2 AB", + "destination.geo.city_name": "Link\u00f6ping", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "SE", + "destination.geo.country_name": "Sweden", + "destination.geo.location.lat": 58.4167, + "destination.geo.location.lon": 15.6167, + "destination.geo.region_iso_code": "SE-E", + "destination.geo.region_name": "\u00d6sterg\u00f6tland County", + "destination.ip": "89.160.20.128", + "destination.port": 2511, + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 106023, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-4-106023: Deny tcp src MG:exp_srv/63513 dst SH_OSS:89.160.20.128/2511 by access-group \"MGT_access_in\" [0x0, 0x0]", + "event.outcome": "success", + "event.severity": 4, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "denied" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "warning", + "log.offset": 1699, + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "SH_OSS", + "observer.hostname": "216.160.83.56", + "observer.ingress.interface.name": "MG", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56", + "exp_srv" + ], + "related.ip": [ + "89.160.20.128" + ], + "service.type": "cisco", + "source.address": "exp_srv", + "source.domain": "exp_srv", + "source.port": 63513, + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.icmp_code": 0, + "cisco.asa.icmp_type": 8, + "cisco.asa.mapped_source_ip": "81.2.69.192", + "cisco.asa.message_id": "302020", + "destination.domain": "eth0_fw", + "event.action": "flow-expiration", + "event.category": [ + "network" + ], + "event.code": 302020, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302020: Built inbound ICMP connection for faddr eth0_fw/6553 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "informational", + "log.offset": 1849, + "network.direction": "inbound", + "network.protocol": "icmp", + "observer.hostname": "81.2.69.142", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "eth0_fw" + ], + "related.ip": [ + "81.2.69.192" + ], + "service.type": "cisco", + "source.address": "81.2.69.192", + "source.geo.city_name": "London", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.country_name": "United Kingdom", + "source.geo.location.lat": 51.5142, + "source.geo.location.lon": -0.0931, + "source.geo.region_iso_code": "GB-ENG", + "source.geo.region_name": "England", + "source.ip": "81.2.69.192", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.icmp_code": 0, + "cisco.asa.icmp_type": 8, + "cisco.asa.mapped_source_ip": "81.2.69.192", + "cisco.asa.message_id": "302020", + "destination.domain": "eth0_fw", + "event.action": "flow-expiration", + "event.category": [ + "network" + ], + "event.code": 302020, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302020: Built inbound ICMP connection for faddr eth0_fw/8396 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "informational", + "log.offset": 1999, + "network.direction": "inbound", + "network.protocol": "icmp", + "observer.hostname": "81.2.69.142", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "eth0_fw" + ], + "related.ip": [ + "81.2.69.192" + ], + "service.type": "cisco", + "source.address": "81.2.69.192", + "source.geo.city_name": "London", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.country_name": "United Kingdom", + "source.geo.location.lat": 51.5142, + "source.geo.location.lon": -0.0931, + "source.geo.region_iso_code": "GB-ENG", + "source.geo.region_name": "England", + "source.ip": "81.2.69.192", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.icmp_code": 0, + "cisco.asa.icmp_type": 8, + "cisco.asa.mapped_source_ip": "81.2.69.192", + "cisco.asa.message_id": "302021", + "destination.domain": "eth0_fw", + "event.action": "flow-expiration", + "event.category": [ + "network" + ], + "event.code": 302021, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302021: Teardown ICMP connection for faddr eth0_fw/6553 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "informational", + "log.offset": 2149, + "network.iana_number": 1, + "network.transport": "icmp", + "observer.hostname": "81.2.69.142", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "eth0_fw" + ], + "related.ip": [ + "81.2.69.192" + ], + "service.type": "cisco", + "source.address": "81.2.69.192", + "source.geo.city_name": "London", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.country_name": "United Kingdom", + "source.geo.location.lat": 51.5142, + "source.geo.location.lon": -0.0931, + "source.geo.region_iso_code": "GB-ENG", + "source.geo.region_name": "England", + "source.ip": "81.2.69.192", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.icmp_code": 0, + "cisco.asa.icmp_type": 8, + "cisco.asa.mapped_source_ip": "81.2.69.192", + "cisco.asa.message_id": "302021", + "destination.domain": "eth0_fw", + "event.action": "flow-expiration", + "event.category": [ + "network" + ], + "event.code": 302021, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302021: Teardown ICMP connection for faddr eth0_fw/8396 gaddr 81.2.69.192/0 laddr 81.2.69.192/0 type 8 code 0", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "connection", + "end" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "informational", + "log.offset": 2294, + "network.iana_number": 1, + "network.transport": "icmp", + "observer.hostname": "81.2.69.142", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "eth0_fw" + ], + "related.ip": [ + "81.2.69.192" + ], + "service.type": "cisco", + "source.address": "81.2.69.192", + "source.geo.city_name": "London", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.country_name": "United Kingdom", + "source.geo.location.lat": 51.5142, + "source.geo.location.lon": -0.0931, + "source.geo.region_iso_code": "GB-ENG", + "source.geo.region_name": "England", + "source.ip": "81.2.69.192", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "113039", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113039, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-113039: Group User IP <216.160.83.56> AnyConnect parent session started.", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.192", + "input.type": "log", + "log.level": "informational", + "log.offset": 2439, + "observer.hostname": "81.2.69.192", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.192" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "113039", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 113039, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-113039: Group User IP <81.2.69.192> AnyConnect parent session started.", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "host.hostname": "216.160.83.56", + "input.type": "log", + "log.level": "informational", + "log.offset": 2566, + "observer.hostname": "216.160.83.56", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "216.160.83.56" + ], + "service.type": "cisco", + "tags": [ + "cisco-asa", + "forwarded" + ] + }, + { + "cisco.asa.message_id": "304001", + "destination.address": "mirror", + "destination.domain": "mirror", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 304001, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-5-304001: 192.168.14.22 Accessed URL mirror:http://mirror.example.com/path/to/resource", + "event.outcome": "success", + "event.severity": 5, + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "fileset.name": "asa", + "host.hostname": "81.2.69.142", + "input.type": "log", + "log.level": "notification", + "log.offset": 2705, + "observer.hostname": "81.2.69.142", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.hosts": [ + "81.2.69.142", + "mirror" + ], + "related.ip": [ + "192.168.14.22" + ], + "service.type": "cisco", + "source.address": "192.168.14.22", + "source.ip": "192.168.14.22", + "tags": [ + "cisco-asa", + "forwarded" + ], + "url.domain": "mirror.example.com", + "url.original": "http://mirror.example.com/path/to/resource", + "url.path": "/path/to/resource", + "url.scheme": "http" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index 460db388419..01a398260fc 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -243,7 +243,7 @@ processors: field: "message" description: "106015" patterns: - - "%{NOTSPACE:event.outcome} %{NOTSPACE:network.transport} %{NOTSPACE} %{NOTSPACE} from %{IP:source.address}/%{POSINT:source.port} to %{IP:destination.address}/%{POSINT:destination.port} flags %{DATA} on interface %{NOTSPACE:_temp_.cisco.source_interface}" + - "%{NOTSPACE:event.outcome} %{NOTSPACE:network.transport} %{NOTSPACE} %{NOTSPACE} from %{IP:source.address}/%{POSINT:source.port} to %{IPORHOST:destination.address}/%{POSINT:destination.port} flags %{DATA} on interface %{NOTSPACE:_temp_.cisco.source_interface}" - dissect: if: "ctx._temp_.cisco.message_id == '106016'" field: "message" @@ -279,8 +279,10 @@ processors: field: "message" description: "106023" patterns: - - ^%{NOTSPACE:event.outcome} ((protocol %{POSINT:network.iana_number})|%{NOTSPACE:network.transport}) src %{NOTCOLON:_temp_.cisco.source_interface}:%{IPORHOST:source.address}(/%{POSINT:source.port})?\s*(\(%{CISCO_USER:_temp_.cisco.source_username}\) )?dst %{NOTCOLON:_temp_.cisco.destination_interface}:%{IPORHOST:destination.address}(/%{POSINT:destination.port})?%{DATA}by access-group "%{NOTSPACE:_temp_.cisco.list_id}" + - ^%{NOTSPACE:event.outcome} ((protocol %{POSINT:network.iana_number})|%{NOTSPACE:network.transport}) src %{NOTCOLON:_temp_.cisco.source_interface}:%{IPORHOST:source.address}(/%{POSINT:source.port})?\s*(\(%{CISCO_USER:_temp_.cisco.source_username}\) )?dst %{NOTCOLON:_temp_.cisco.destination_interface}:%{IPORHOST:destination.address}(/%{POSINT:destination.port})?%{DATA}by access-group "%{NOTSPACE:_temp_.cisco.list_id}" pattern_definitions: + HOSTNAME: "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\\.?|\\b)" + IPORHOST: "(?:%{IP}|%{HOSTNAME})" NOTCOLON: "[^:]*" CISCO_USER: ((LOCAL\\)?(%{HOSTNAME}\\)?%{USERNAME}(@%{HOSTNAME})?(, *%{NUMBER})?) - dissect: @@ -343,8 +345,10 @@ processors: field: "message" description: "302013, 302015" patterns: - - Built %{NOTSPACE:network.direction} %{NOTSPACE:network.transport} connection %{NUMBER:_temp_.cisco.connection_id} for %{NOTCOLON:_temp_.cisco.source_interface}:%{IP:source.address}/%{NUMBER:source.port} \(%{IP:_temp_.natsrcip}/%{NUMBER:_temp_.cisco.mapped_source_port}\)(\(%{CISCO_USER:_temp_.cisco.source_username}\))? to %{NOTCOLON:_temp_.cisco.destination_interface}:%{NOTSPACE:destination.address}/%{NUMBER:destination.port} \(%{NOTSPACE:_temp_.natdstip}/%{NUMBER:_temp_.cisco.mapped_destination_port}\)(\(%{CISCO_USER:_temp_.cisco.destination_username}\))?( \(%{CISCO_USER:_temp_.cisco.termination_user}\))?%{GREEDYDATA} + - Built %{NOTSPACE:network.direction} %{NOTSPACE:network.transport} connection %{NUMBER:_temp_.cisco.connection_id} for %{NOTCOLON:_temp_.cisco.source_interface}:%{IPORHOST:source.address}/%{NUMBER:source.port} \(%{IPORHOST:_temp_.natsrcip}/%{NUMBER:_temp_.cisco.mapped_source_port}\)(\(%{CISCO_USER:_temp_.cisco.source_username}\))? to %{NOTCOLON:_temp_.cisco.destination_interface}:%{NOTSPACE:destination.address}/%{NUMBER:destination.port} \(%{NOTSPACE:_temp_.natdstip}/%{NUMBER:_temp_.cisco.mapped_destination_port}\)(\(%{CISCO_USER:_temp_.cisco.destination_username}\))?( \(%{CISCO_USER:_temp_.cisco.termination_user}\))?%{GREEDYDATA} pattern_definitions: + HOSTNAME: "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\\.?|\\b)" + IPORHOST: "(?:%{IP}|%{HOSTNAME})" NOTCOLON: "[^:]*" CISCO_USER: ((LOCAL\\)?(%{HOSTNAME}\\)?%{USERNAME}(@%{HOSTNAME})?(, *%{NUMBER})?) - dissect: @@ -357,9 +361,11 @@ processors: field: "message" description: "305012" patterns: - - Teardown %{DATA} %{NOTSPACE:network.transport} translation from %{NOTCOLON:_temp_.cisco.source_interface}:%{IP:source.address}/%{NUMBER:source.port}(\s*\(%{CISCO_USER:_temp_.cisco.source_username}\))? to %{NOTCOLON:_temp_.cisco.destination_interface}:%{IP:destination.address}/%{NUMBER:destination.port} duration %{DURATION:_temp_.duration_hms} + - Teardown %{DATA} %{NOTSPACE:network.transport} translation from %{NOTCOLON:_temp_.cisco.source_interface}:%{IPORHOST:source.address}/%{NUMBER:source.port}(\s*\(%{CISCO_USER:_temp_.cisco.source_username}\))? to %{NOTCOLON:_temp_.cisco.destination_interface}:%{IP:destination.address}/%{NUMBER:destination.port} duration %{DURATION:_temp_.duration_hms} pattern_definitions: NOTCOLON: "[^:]*" + HOSTNAME: "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\\.?|\\b)" + IPORHOST: "(?:%{IP}|%{HOSTNAME})" CISCO_USER: ((LOCAL\\)?(%{HOSTNAME}\\)?%{USERNAME}(@%{HOSTNAME})?(, *%{NUMBER})?) DURATION: "%{INT}:%{MINUTE}:%{SECOND}" - grok: @@ -367,8 +373,10 @@ processors: field: "message" description: "302020" patterns: - - "Built %{NOTSPACE:network.direction} %{NOTSPACE:network.protocol} connection for faddr (?:%{NOTCOLON:_temp_.cisco.source_interface}:)?%{ECSDESTIPORHOST}/%{NUMBER}\\s*(?:\\(%{CISCO_USER:_temp_.cisco.destination_username}\\) )?gaddr (?:%{NOTCOLON}:)?%{MAPPEDSRC}/%{NUMBER} laddr (?:%{NOTCOLON:_temp_.cisco.source_interface}:)?%{ECSSOURCEIPORHOST}/%{NUMBER}\\s*(?:\\(%{CISCO_USER:_temp_.cisco.source_username}\\) )?(type %{NUMBER:_temp_.cisco.icmp_type} code %{NUMBER:_temp_.cisco.icmp_code})?" + - "Built %{NOTSPACE:network.direction} %{NOTSPACE:network.protocol} connection for faddr (?:%{NOTCOLON:_temp_.cisco.source_interface}:)?%{ECSDESTIPORHOST}/%{NUMBER}\\s*(?:\\(%{CISCO_USER:_temp_.cisco.destination_username}\\) )?gaddr (?:%{NOTCOLON}:)?%{MAPPEDSRC}/%{NUMBER} laddr (?:%{NOTCOLON:_temp_.cisco.source_interface}:)?%{ECSSOURCEIPORHOST}/%{NUMBER}\\s*(?:\\(%{CISCO_USER:_temp_.cisco.source_username}\\) )?(type %{NUMBER:_temp_.cisco.icmp_type} code %{NUMBER:_temp_.cisco.icmp_code})?" pattern_definitions: + HOSTNAME: "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\\.?|\\b)" + IPORHOST: "(?:%{IP}|%{HOSTNAME})" NOTCOLON: "[^:]*" ECSSOURCEIPORHOST: "(?:%{IP:source.address}|%{HOSTNAME:source.domain})" ECSDESTIPORHOST: "(?:%{IP:destination.address}|%{HOSTNAME:destination.domain})" @@ -389,12 +397,12 @@ processors: field: "message" description: "304001" patterns: - - "(%{NOTSPACE:source.user.name}@)?%{IP:source.address}(\\(%{DATA}\\))? %{DATA} (%{NOTSPACE}@)?%{IP:destination.address}:%{GREEDYDATA:url.original}" + - "(%{NOTSPACE:source.user.name}@)?%{IP:source.address}(\\(%{DATA}\\))? %{DATA} (%{NOTSPACE}@)?%{IPORHOST:destination.address}:%{GREEDYDATA:url.original}" - set: if: "ctx._temp_.cisco.message_id == '304001'" field: "event.outcome" description: "304001" - value: "allowed" + value: allowed - dissect: if: "ctx._temp_.cisco.message_id == '304002'" field: "message" @@ -405,7 +413,7 @@ processors: field: "message" description: "305011" patterns: - - Built %{NOTSPACE} %{NOTSPACE:network.transport} translation from %{NOTSPACE:_temp_.cisco.source_interface}:%{IP:source.address}/%{NUMBER:source.port}(\(%{NOTSPACE:source.user.name}\))? to %{NOTSPACE:_temp_.cisco.destination_interface}:%{IP:destination.address}/%{NUMBER:destination.port} + - Built %{NOTSPACE} %{NOTSPACE:network.transport} translation from %{NOTSPACE:_temp_.cisco.source_interface}:%{IPORHOST:source.address}/%{NUMBER:source.port}(\(%{NOTSPACE:source.user.name}\))? to %{NOTSPACE:_temp_.cisco.destination_interface}:%{IP:destination.address}/%{NUMBER:destination.port} - dissect: if: "ctx._temp_.cisco.message_id == '313001'" field: "message" @@ -822,6 +830,8 @@ processors: - ^Teardown %{NOTSPACE:network.transport} (?:state-bypass )?connection %{NOTSPACE:_temp_.cisco.connection_id} (?:for|from) %{NOTCOLON:_temp_.cisco.source_interface}:%{DATA:source.address}/%{NUMBER:source.port:int}\s*(?:\(?%{CISCO_USER:_temp_.cisco.source_username}\)? )?to %{NOTCOLON:_temp_.cisco.destination_interface}:%{DATA:destination.address}/%{NUMBER:destination.port:int}\s*(?:\(?%{CISCO_USER:_temp_.cisco.destination_username}\)? )?duration (?:%{DURATION:_temp_.duration_hms} bytes %{NUMBER:network.bytes}) - ^Teardown %{NOTSPACE:network.transport} connection for faddr (?:%{NOTCOLON:_temp_.cisco.source_interface}:)?%{ECSDESTIPORHOST}/%{NUMBER}\s*(?:\(?%{CISCO_USER:_temp_.cisco.destination_username}\)? )?gaddr (?:%{NOTCOLON}:)?%{MAPPEDSRC}/%{NUMBER} laddr (?:%{NOTCOLON:_temp_.cisco.source_interface}:)?%{ECSSOURCEIPORHOST}/%{NUMBER}\s*(?:\(%{CISCO_USER:_temp_.cisco.source_username}\))?(\s*type %{NUMBER:_temp_.cisco.icmp_type} code %{NUMBER:_temp_.cisco.icmp_code})? pattern_definitions: + HOSTNAME: "\\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\\.?|\\b)" + IPORHOST: "(?:%{IP}|%{HOSTNAME})" NOTCOLON: "[^:]*" ECSSOURCEIPORHOST: "(?:%{IP:source.address}|%{HOSTNAME:source.domain})" ECSDESTIPORHOST: "(?:%{IP:destination.address}|%{HOSTNAME:destination.domain})" diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef.go b/x-pack/filebeat/processors/decode_cef/decode_cef.go index 67b6634712a..8e859740080 100644 --- a/x-pack/filebeat/processors/decode_cef/decode_cef.go +++ b/x-pack/filebeat/processors/decode_cef/decode_cef.go @@ -6,10 +6,10 @@ package decode_cef import ( "encoding/json" + "fmt" "strconv" "strings" - "github.com/pkg/errors" "go.uber.org/multierr" "github.com/elastic/beats/v7/libbeat/beat" @@ -38,7 +38,7 @@ type processor struct { func New(cfg *conf.C) (processors.Processor, error) { c := defaultConfig() if err := cfg.Unpack(&c); err != nil { - return nil, errors.Wrap(err, "fail to unpack the "+procName+" processor configuration") + return nil, fmt.Errorf("fail to unpack the "+procName+" processor configuration: %w", err) } return newDecodeCEF(c) @@ -64,7 +64,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { if p.IgnoreMissing { return event, nil } - return event, errors.Wrapf(err, "decode_cef field [%v] not found", p.Field) + return event, fmt.Errorf("decode_cef field [%v] not found: %w", p.Field, err) } cefData, ok := v.(string) @@ -72,7 +72,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { if p.IgnoreFailure { return event, nil } - return event, errors.Wrapf(err, "decode_cef field [%v] is not a string", p.Field) + return event, fmt.Errorf("decode_cef field [%v] is not a string: %T", p.Field, v) } // Ignore any leading data before the CEF header. @@ -81,7 +81,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { if p.IgnoreFailure { return event, nil } - return event, errors.Errorf("decode_cef field [%v] does not contain a CEF header", p.Field) + return event, fmt.Errorf("decode_cef field [%v] does not contain a CEF header", p.Field) } cefData = cefData[idx:] @@ -91,12 +91,15 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { if p.IgnoreFailure { return event, nil } - return event, errors.Wrap(err, "decode_cef failed to parse message") + if err != nil { + err = fmt.Errorf("decode_cef failed to parse message: %w", err) + } + return event, err } cefErrors := multierr.Errors(err) cefObject := toCEFObject(&ce) - event.PutValue(p.TargetField, cefObject) + _, _ = event.PutValue(p.TargetField, cefObject) // Map CEF extension fields to ECS fields. if p.ECS { @@ -112,16 +115,16 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { if mapping.Translate != nil { translatedValue, err := mapping.Translate(field) if err != nil { - cefErrors = append(cefErrors, errors.Wrap(err, key)) + cefErrors = append(cefErrors, fmt.Errorf("%s: %w", key, err)) continue } if translatedValue != nil { - event.PutValue(mapping.Target, translatedValue) + _, _ = event.PutValue(mapping.Target, translatedValue) } } else if field.Interface != nil { - event.PutValue(mapping.Target, field.Interface) + _, _ = event.PutValue(mapping.Target, field.Interface) } else { - event.PutValue(mapping.Target, field.String) + _, _ = event.PutValue(mapping.Target, field.String) } } } @@ -137,6 +140,7 @@ func (p *processor) Run(event *beat.Event) (*beat.Event, error) { return event, nil } +//nolint:errcheck // All errors are from mapstr puts. func toCEFObject(cefEvent *cef.Event) mapstr.M { // Add CEF header fields. cefObject := mapstr.M{"version": strconv.Itoa(cefEvent.Version)} @@ -175,6 +179,7 @@ func toCEFObject(cefEvent *cef.Event) mapstr.M { return cefObject } +//nolint:errcheck // All errors are from mapstr puts. func writeCEFHeaderToECS(cefEvent *cef.Event, event *beat.Event) { if cefEvent.DeviceVendor != "" { event.PutValue("observer.vendor", cefEvent.DeviceVendor) @@ -199,6 +204,7 @@ func writeCEFHeaderToECS(cefEvent *cef.Event, event *beat.Event) { } } +//nolint:errcheck // All errors are from mapstr puts. func appendErrorMessage(m mapstr.M, msg string) error { const field = "error.message" list, _ := m.GetValue(field) @@ -227,7 +233,7 @@ func appendErrorMessage(m mapstr.M, msg string) error { } m.Put(field, append(v, msg)) default: - return errors.Errorf("unexpected type %T found for %v field", list, field) + return fmt.Errorf("unexpected type %T found for %v field", list, field) } return nil } diff --git a/x-pack/filebeat/processors/decode_cef/keys.ecs.go b/x-pack/filebeat/processors/decode_cef/keys.ecs.go index e8056a1ac32..a9fc28159f7 100644 --- a/x-pack/filebeat/processors/decode_cef/keys.ecs.go +++ b/x-pack/filebeat/processors/decode_cef/keys.ecs.go @@ -5,38 +5,51 @@ package decode_cef import ( + "errors" "strings" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/x-pack/filebeat/processors/decode_cef/cef" ) type mappedField struct { - Target string + // Target is the ECS target field for the mapped field. + Target string + + // Translate is the mapping function required to translate + // the CEF field data into an ECS-conformant format. + // If Translate is nil, no translation is done. + // Translate should not mutate the input and should + // return an error if the input data cannot be correctly + // mapped to ECS-formatted data for the target field. Translate func(in *cef.Field) (interface{}, error) } var ecsExtensionMapping = map[string]mappedField{ - "agentAddress": {Target: "agent.ip"}, - "agentDnsDomain": {Target: "agent.name"}, - "agentHostName": {Target: "agent.name"}, - "agentId": {Target: "agent.id"}, - "agentMacAddress": {Target: "agent.mac"}, - "agentReceiptTime": {Target: "event.created"}, - "agentType": {Target: "agent.type"}, - "agentVersion": {Target: "agent.version"}, - "applicationProtocol": {Target: "network.application"}, - "bytesIn": {Target: "source.bytes"}, - "bytesOut": {Target: "destination.bytes"}, - "customerExternalID": {Target: "organization.id"}, - "customerURI": {Target: "organization.name"}, - "destinationAddress": {Target: "destination.ip"}, - "destinationDnsDomain": {Target: "destination.domain"}, - "destinationGeoLatitude": {Target: "destination.geo.location.lat"}, - "destinationGeoLongitude": {Target: "destination.geo.location.lon"}, - "destinationHostName": {Target: "destination.domain"}, - "destinationMacAddress": {Target: "destination.mac"}, + "agentAddress": {Target: "agent.ip"}, + "agentDnsDomain": {Target: "agent.name"}, + "agentHostName": {Target: "agent.name"}, + "agentId": {Target: "agent.id"}, + "agentMacAddress": { + Target: "agent.mac", + Translate: ecsMAC, + }, + "agentReceiptTime": {Target: "event.created"}, + "agentType": {Target: "agent.type"}, + "agentVersion": {Target: "agent.version"}, + "applicationProtocol": {Target: "network.application"}, + "bytesIn": {Target: "source.bytes"}, + "bytesOut": {Target: "destination.bytes"}, + "customerExternalID": {Target: "organization.id"}, + "customerURI": {Target: "organization.name"}, + "destinationAddress": {Target: "destination.ip"}, + "destinationDnsDomain": {Target: "destination.domain"}, + "destinationGeoLatitude": {Target: "destination.geo.location.lat"}, + "destinationGeoLongitude": {Target: "destination.geo.location.lon"}, + "destinationHostName": {Target: "destination.domain"}, + "destinationMacAddress": { + Target: "destination.mac", + Translate: ecsMAC, + }, "destinationPort": {Target: "destination.port"}, "destinationProcessId": {Target: "destination.process.pid"}, "destinationProcessName": {Target: "destination.process.name"}, @@ -57,13 +70,16 @@ var ecsExtensionMapping = map[string]mappedField{ case "1": return "outbound", nil default: - return nil, errors.Errorf("deviceDirection must be 0 or 1") + return nil, errors.New("deviceDirection must be 0 or 1") } }, }, - "deviceDnsDomain": {Target: "observer.hostname"}, - "deviceHostName": {Target: "observer.hostname"}, - "deviceMacAddress": {Target: "observer.mac"}, + "deviceDnsDomain": {Target: "observer.hostname"}, + "deviceHostName": {Target: "observer.hostname"}, + "deviceMacAddress": { + Target: "observer.mac", + Translate: ecsMAC, + }, "devicePayloadId": {Target: "event.id"}, "deviceProcessId": {Target: "process.pid"}, "deviceProcessName": {Target: "process.name"}, @@ -92,14 +108,17 @@ var ecsExtensionMapping = map[string]mappedField{ return nil, nil }, }, - "requestMethod": {Target: "http.request.method"}, - "requestUrl": {Target: "url.original"}, - "sourceAddress": {Target: "source.ip"}, - "sourceDnsDomain": {Target: "source.domain"}, - "sourceGeoLatitude": {Target: "source.geo.location.lat"}, - "sourceGeoLongitude": {Target: "source.geo.location.lon"}, - "sourceHostName": {Target: "source.domain"}, - "sourceMacAddress": {Target: "source.mac"}, + "requestMethod": {Target: "http.request.method"}, + "requestUrl": {Target: "url.original"}, + "sourceAddress": {Target: "source.ip"}, + "sourceDnsDomain": {Target: "source.domain"}, + "sourceGeoLatitude": {Target: "source.geo.location.lat"}, + "sourceGeoLongitude": {Target: "source.geo.location.lon"}, + "sourceHostName": {Target: "source.domain"}, + "sourceMacAddress": { + Target: "source.mac", + Translate: ecsMAC, + }, "sourcePort": {Target: "source.port"}, "sourceProcessId": {Target: "source.process.pid"}, "sourceProcessName": {Target: "source.process.name"}, @@ -118,3 +137,7 @@ var ecsExtensionMapping = map[string]mappedField{ }, "type": {Target: "event.kind"}, } + +func ecsMAC(in *cef.Field) (interface{}, error) { + return strings.ToUpper(strings.ReplaceAll(in.String, ":", "-")), nil +} diff --git a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json index 3f1a1c38398..b7895363aec 100644 --- a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json +++ b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json @@ -220,7 +220,7 @@ "destination": { "bytes": 1019, "ip": "72.14.204.147", - "mac": "00:50:56:f5:7f:47", + "mac": "00-50-56-F5-7F-47", "port": 80 }, "error": { @@ -243,7 +243,7 @@ }, "source": { "ip": "192.168.126.150", - "mac": "00:0c:29:eb:35:de", + "mac": "00-0C-29-EB-35-DE", "port": 49617 } }, @@ -332,7 +332,7 @@ "destination": { "bytes": 1093, "ip": "72.14.204.105", - "mac": "00:50:56:f5:7f:47", + "mac": "00-50-56-F5-7F-47", "port": 80 }, "event": { @@ -353,7 +353,7 @@ }, "source": { "ip": "192.168.126.150", - "mac": "00:0c:29:eb:35:de", + "mac": "00-0C-29-EB-35-DE", "port": 49786 } }, @@ -800,7 +800,7 @@ "agent": { "id": "4SNQXV30BABCAIi+-ZH3gxT==", "ip": "10.2.3.4", - "mac": "00:50:56:8e:c0:90", + "mac": "00-50-56-8E-C0-90", "name": "centos7.as", "type": "syslog", "version": "7.6.0.8009.0" diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 0d997dbcbcd..27c7d510652 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.12 +FROM golang:1.18.5 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index dcca85bfd70..9f410f950ed 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,12 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/functionbeat check; - make -C x-pack/functionbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml index af4efc607c2..c5df1f51d77 100644 --- a/x-pack/heartbeat/Jenkinsfile.yml +++ b/x-pack/heartbeat/Jenkinsfile.yml @@ -13,19 +13,13 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/heartbeat check; - make -C x-pack/heartbeat update; - make -C heartbeat check; - make -C heartbeat update; - make check-no-changes; - stage: checks unitTest: mage: "mage build unitTest" + withNodejs: true stage: mandatory goIntegTest: mage: "mage goIntegTest" + withNodejs: true stage: mandatory ironbank: mage: "mage ironbank" diff --git a/x-pack/heartbeat/import.go b/x-pack/heartbeat/import.go new file mode 100644 index 00000000000..c23e267b6ce --- /dev/null +++ b/x-pack/heartbeat/import.go @@ -0,0 +1,12 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux || darwin +// +build linux darwin + +package main + +import ( + _ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser" +) diff --git a/x-pack/heartbeat/main.go b/x-pack/heartbeat/main.go index 4c0f220edf6..2cd061dff30 100644 --- a/x-pack/heartbeat/main.go +++ b/x-pack/heartbeat/main.go @@ -9,7 +9,6 @@ import ( _ "github.com/elastic/beats/v7/heartbeat/include" "github.com/elastic/beats/v7/x-pack/heartbeat/cmd" - _ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser" ) func main() { diff --git a/x-pack/heartbeat/monitors.d/browser-inline.yml b/x-pack/heartbeat/monitors.d/browser-inline.yml deleted file mode 100644 index b5a70f05050..00000000000 --- a/x-pack/heartbeat/monitors.d/browser-inline.yml +++ /dev/null @@ -1,18 +0,0 @@ -- type: browser - enabled: true - id: cg-test-inl - name: CG TESTINLINE - data_stream.namespace: my-namespace - source: - inline: - script: - step("load homepage", async () => { - await page.goto('https://www.elastic.co'); - }); - step("hover over products menu", async () => { - await page.hover('css=[data-nav-item=products]'); - }); - step("failme", async () => { - await page.hhover('css=[data-nav-item=products]'); - }); - schedule: "@every 1m" diff --git a/x-pack/heartbeat/monitors.d/todos.yml.disabled b/x-pack/heartbeat/monitors.d/todos.yml.disabled deleted file mode 100644 index cad26367d37..00000000000 --- a/x-pack/heartbeat/monitors.d/todos.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -- name: CGX Todos - id: cgx-todos - type: browser - schedule: "@every 3m" - tags: todos-app - params: - url: "https://elastic.github.io/synthetics-demo/" - source: - zip_url: - url: "https://github.com/elastic/synthetics-demo/archive/refs/heads/main.zip" - folder: "todos/synthetics-tests" diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go index 3bdcdbbd962..a84422b4b82 100644 --- a/x-pack/heartbeat/monitors/browser/browser.go +++ b/x-pack/heartbeat/monitors/browser/browser.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package browser @@ -10,9 +12,10 @@ import ( "sync" "syscall" - "github.com/elastic/beats/v7/heartbeat/monitors/plugin" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/beats/v7/heartbeat/monitors/plugin" ) func init() { diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go index 4b085c11c69..0f0d1401ec4 100644 --- a/x-pack/heartbeat/monitors/browser/config.go +++ b/x-pack/heartbeat/monitors/browser/config.go @@ -2,10 +2,14 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin + package browser import ( "fmt" + "time" "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source" "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec" @@ -16,6 +20,7 @@ func DefaultConfig() *Config { return &Config{ Sandbox: false, Screenshots: "on", + Timeout: 15 * time.Minute, } } @@ -35,6 +40,7 @@ type Config struct { PlaywrightOpts map[string]interface{} `config:"playwright_options"` FilterJourneys synthexec.FilterJourneyConfig `config:"filter_journeys"` IgnoreHTTPSErrors bool `config:"ignore_https_errors"` + Timeout time.Duration `config:"timeout"` } var ErrNameRequired = fmt.Errorf("config 'name' must be specified for this monitor") diff --git a/x-pack/heartbeat/monitors/browser/config_test.go b/x-pack/heartbeat/monitors/browser/config_test.go index f1a8cab8565..460fc73bbe0 100644 --- a/x-pack/heartbeat/monitors/browser/config_test.go +++ b/x-pack/heartbeat/monitors/browser/config_test.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package browser diff --git a/x-pack/heartbeat/monitors/browser/project.go b/x-pack/heartbeat/monitors/browser/project.go index 2e7fd61699f..b7b37f0bb20 100644 --- a/x-pack/heartbeat/monitors/browser/project.go +++ b/x-pack/heartbeat/monitors/browser/project.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package browser @@ -8,6 +10,7 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/plugin" @@ -125,16 +128,18 @@ func (p *Project) extraArgs() []string { func (p *Project) jobs() []jobs.Job { var j jobs.Job isScript := p.projectCfg.Source.Inline != nil + ctx := context.WithValue(context.Background(), synthexec.SynthexecTimeout, p.projectCfg.Timeout+30*time.Second) + if isScript { src := p.projectCfg.Source.Inline.Script - j = synthexec.InlineJourneyJob(context.TODO(), src, p.Params(), p.StdFields(), p.extraArgs()...) + j = synthexec.InlineJourneyJob(ctx, src, p.Params(), p.StdFields(), p.extraArgs()...) } else { j = func(event *beat.Event) ([]jobs.Job, error) { err := p.Fetch() if err != nil { return nil, fmt.Errorf("could not fetch for project job: %w", err) } - sj, err := synthexec.ProjectJob(context.TODO(), p.Workdir(), p.Params(), p.FilterJourneys(), p.StdFields(), p.extraArgs()...) + sj, err := synthexec.ProjectJob(ctx, p.Workdir(), p.Params(), p.FilterJourneys(), p.StdFields(), p.extraArgs()...) if err != nil { return nil, err } diff --git a/x-pack/heartbeat/monitors/browser/project_test.go b/x-pack/heartbeat/monitors/browser/project_test.go index d3a03c13280..9a89bdabcd7 100644 --- a/x-pack/heartbeat/monitors/browser/project_test.go +++ b/x-pack/heartbeat/monitors/browser/project_test.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package browser @@ -21,6 +23,7 @@ import ( ) func TestValidLocal(t *testing.T) { + timeout := 30 _, filename, _, _ := runtime.Caller(0) path := path.Join(filepath.Dir(filename), "source/fixtures/todos") testParams := map[string]interface{}{ @@ -40,6 +43,7 @@ func TestValidLocal(t *testing.T) { "path": path, }, }, + "timeout": timeout, }) s, e := NewProject(cfg) require.NoError(t, e) @@ -57,6 +61,7 @@ func TestValidLocal(t *testing.T) { } func TestValidInline(t *testing.T) { + timeout := 30 script := "a script" testParams := map[string]interface{}{ "key1": "value1", @@ -71,6 +76,7 @@ func TestValidInline(t *testing.T) { "script": script, }, }, + "timeout": timeout, }) s, e := NewProject(cfg) require.NoError(t, e) @@ -211,3 +217,21 @@ func TestExtraArgs(t *testing.T) { }) } } + +func TestEmptyTimeout(t *testing.T) { + defaults := DefaultConfig() + cfg := conf.MustNewConfigFrom(mapstr.M{ + "name": "My Name", + "id": "myId", + "source": mapstr.M{ + "inline": mapstr.M{ + "script": "script", + }, + }, + }) + s, e := NewProject(cfg) + + require.NoError(t, e) + require.NotNil(t, s) + require.Equal(t, s.projectCfg.Timeout, defaults.Timeout) +} diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go index b27f26e4438..ebaf628214d 100644 --- a/x-pack/heartbeat/monitors/browser/source/local.go +++ b/x-pack/heartbeat/monitors/browser/source/local.go @@ -56,7 +56,7 @@ func (l *LocalSource) Fetch() (err error) { if l.workingPath != "" { return nil } - l.workingPath, err = ioutil.TempDir("/tmp", "elastic-synthetics-") + l.workingPath, err = ioutil.TempDir(os.TempDir(), "elastic-synthetics-") if err != nil { return fmt.Errorf("could not create tmp dir: %w", err) } diff --git a/x-pack/heartbeat/monitors/browser/source/project.go b/x-pack/heartbeat/monitors/browser/source/project.go index 631809484bc..effea44eb8a 100644 --- a/x-pack/heartbeat/monitors/browser/source/project.go +++ b/x-pack/heartbeat/monitors/browser/source/project.go @@ -41,7 +41,7 @@ func (p *ProjectSource) Fetch() error { return err } - tf, err := ioutil.TempFile("/tmp", "elastic-synthetics-zip-") + tf, err := ioutil.TempFile(os.TempDir(), "elastic-synthetics-zip-") if err != nil { return fmt.Errorf("could not create tmpfile for project monitor source: %w", err) } @@ -53,7 +53,7 @@ func (p *ProjectSource) Fetch() error { return err } - p.TargetDirectory, err = ioutil.TempDir("/tmp", "elastic-synthetics-unzip-") + p.TargetDirectory, err = ioutil.TempDir(os.TempDir(), "elastic-synthetics-unzip-") if err != nil { return fmt.Errorf("could not make temp dir for unzipping project source: %w", err) } diff --git a/x-pack/heartbeat/monitors/browser/source/source.go b/x-pack/heartbeat/monitors/browser/source/source.go index adcc7b17159..b597a045885 100644 --- a/x-pack/heartbeat/monitors/browser/source/source.go +++ b/x-pack/heartbeat/monitors/browser/source/source.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package source diff --git a/x-pack/heartbeat/monitors/browser/source/source_test.go b/x-pack/heartbeat/monitors/browser/source/source_test.go index f8e9e687915..63e19adef78 100644 --- a/x-pack/heartbeat/monitors/browser/source/source_test.go +++ b/x-pack/heartbeat/monitors/browser/source/source_test.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin + package source import ( diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl.go b/x-pack/heartbeat/monitors/browser/source/zipurl.go index 693b232292e..ebd5a1c547f 100644 --- a/x-pack/heartbeat/monitors/browser/source/zipurl.go +++ b/x-pack/heartbeat/monitors/browser/source/zipurl.go @@ -60,7 +60,7 @@ func (z *ZipURLSource) Fetch() error { os.RemoveAll(z.TargetDirectory) } - tf, err := ioutil.TempFile("/tmp", "elastic-synthetics-zip-") + tf, err := ioutil.TempFile(os.TempDir(), "elastic-synthetics-zip-") if err != nil { return fmt.Errorf("could not create tmpfile for zip source: %w", err) } @@ -79,7 +79,7 @@ func (z *ZipURLSource) Fetch() error { return fmt.Errorf("could not make directory %s: %w", z.TargetDirectory, err) } } else { - z.TargetDirectory, err = ioutil.TempDir("/tmp", "elastic-synthetics-unzip-") + z.TargetDirectory, err = ioutil.TempDir(os.TempDir(), "elastic-synthetics-unzip-") if err != nil { return fmt.Errorf("could not make temp dir for zip download: %w", err) } diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl_test.go b/x-pack/heartbeat/monitors/browser/source/zipurl_test.go index db482bed20c..6f6da1f159a 100644 --- a/x-pack/heartbeat/monitors/browser/source/zipurl_test.go +++ b/x-pack/heartbeat/monitors/browser/source/zipurl_test.go @@ -44,7 +44,7 @@ func TestSimpleCases(t *testing.T) { mapstr.M{ "folder": "/", "retries": 3, - "target_directory": "/tmp/synthetics/blah", + "target_directory": filepath.Join(os.TempDir(), "synthetics", "blah"), }, false, false, diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go index 7bb20a06788..6e98e26fba8 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec @@ -23,12 +25,13 @@ import ( type enricher func(event *beat.Event, se *SynthEvent) error type streamEnricher struct { - je *journeyEnricher - sFields stdfields.StdMonitorFields + je *journeyEnricher + sFields stdfields.StdMonitorFields + checkGroup string } func newStreamEnricher(sFields stdfields.StdMonitorFields) *streamEnricher { - return &streamEnricher{sFields: sFields} + return &streamEnricher{sFields: sFields, checkGroup: makeUuid()} } func (senr *streamEnricher) enrich(event *beat.Event, se *SynthEvent) error { @@ -36,6 +39,8 @@ func (senr *streamEnricher) enrich(event *beat.Event, se *SynthEvent) error { senr.je = newJourneyEnricher(senr) } + eventext.MergeEventFields(event, map[string]interface{}{"monitor": map[string]interface{}{"check_group": senr.checkGroup}}) + return senr.je.enrich(event, se) } @@ -44,7 +49,6 @@ func (senr *streamEnricher) enrich(event *beat.Event, se *SynthEvent) error { type journeyEnricher struct { journeyComplete bool journey *Journey - checkGroup string errorCount int error error stepCount int @@ -58,7 +62,6 @@ type journeyEnricher struct { func newJourneyEnricher(senr *streamEnricher) *journeyEnricher { return &journeyEnricher{ - checkGroup: makeUuid(), streamEnricher: senr, } } @@ -82,7 +85,6 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error { switch se.Type { case JourneyStart: je.error = nil - je.checkGroup = makeUuid() je.journey = se.Journey je.start = event.Timestamp case JourneyEnd, CmdStatus: @@ -114,9 +116,8 @@ func (je *journeyEnricher) enrichSynthEvent(event *beat.Event, se *SynthEvent) e if je.journey != nil { eventext.MergeEventFields(event, mapstr.M{ "monitor": mapstr.M{ - "check_group": je.checkGroup, - "id": je.journey.ID, - "name": je.journey.Name, + "id": je.journey.ID, + "name": je.journey.Name, }, }) } @@ -172,7 +173,7 @@ func (je *journeyEnricher) createSummary(event *beat.Event) error { down = 0 } - // Incase of syntax errors or incorrect runner options, the Synthetics + // In case of syntax errors or incorrect runner options, the Synthetics // runner would exit immediately with exitCode 1 and we do not set the duration // to inform the journey never ran if !je.start.IsZero() { @@ -206,6 +207,9 @@ func (je *journeyEnricher) createSummary(event *beat.Event) error { return je.error } + // create a new check group for the next journey + je.streamEnricher.checkGroup = makeUuid() + return fmt.Errorf("journey did not finish executing, %d steps ran: %w", je.stepCount, je.error) } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go index 83eed3e0a55..9f8a47da3db 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec @@ -111,7 +113,9 @@ func TestJourneyEnricher(t *testing.T) { check := func(t *testing.T, se *SynthEvent, je *journeyEnricher) { e := &beat.Event{} t.Run(fmt.Sprintf("event: %s", se.Type), func(t *testing.T) { - enrichErr := je.enrich(e, se) + // we invoke the stream enricher's enrich function, which in turn calls the journey enricher + // we do this because we want the check group set + enrichErr := je.streamEnricher.enrich(e, se) if se.Error != nil { require.Equal(t, stepError(se.Error), enrichErr) } @@ -151,13 +155,11 @@ func TestJourneyEnricher(t *testing.T) { func TestEnrichConsoleSynthEvents(t *testing.T) { tests := []struct { name string - je *journeyEnricher se *SynthEvent check func(t *testing.T, e *beat.Event, je *journeyEnricher) }{ { "stderr", - &journeyEnricher{}, &SynthEvent{ Type: Stderr, Payload: mapstr.M{ @@ -181,7 +183,6 @@ func TestEnrichConsoleSynthEvents(t *testing.T) { }, { "stdout", - &journeyEnricher{}, &SynthEvent{ Type: Stdout, Payload: mapstr.M{ @@ -208,9 +209,10 @@ func TestEnrichConsoleSynthEvents(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { e := &beat.Event{} - err := tt.je.enrichSynthEvent(e, tt.se) + je := newJourneyEnricher(newStreamEnricher(stdfields.StdMonitorFields{})) + err := je.enrichSynthEvent(e, tt.se) require.NoError(t, err) - tt.check(t, e, tt.je) + tt.check(t, e, je) }) } } @@ -218,14 +220,12 @@ func TestEnrichConsoleSynthEvents(t *testing.T) { func TestEnrichSynthEvent(t *testing.T) { tests := []struct { name string - je *journeyEnricher se *SynthEvent wantErr bool check func(t *testing.T, e *beat.Event, je *journeyEnricher) }{ { "cmd/status - with error", - &journeyEnricher{}, &SynthEvent{ Type: CmdStatus, Error: &SynthError{Name: "cmdexit", Message: "cmd err msg"}, @@ -245,7 +245,6 @@ func TestEnrichSynthEvent(t *testing.T) { // If a journey did not emit `journey/end` but exited without // errors, we consider the journey to be up. "cmd/status - without error", - &journeyEnricher{}, &SynthEvent{ Type: CmdStatus, Error: nil, @@ -263,7 +262,6 @@ func TestEnrichSynthEvent(t *testing.T) { }, { "journey/end", - &journeyEnricher{}, &SynthEvent{Type: JourneyEnd}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { @@ -278,7 +276,6 @@ func TestEnrichSynthEvent(t *testing.T) { }, { "step/end", - &journeyEnricher{}, &SynthEvent{Type: "step/end"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { @@ -287,7 +284,6 @@ func TestEnrichSynthEvent(t *testing.T) { }, { "step/screenshot", - &journeyEnricher{}, &SynthEvent{Type: "step/screenshot"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { @@ -296,7 +292,6 @@ func TestEnrichSynthEvent(t *testing.T) { }, { "step/screenshot_ref", - &journeyEnricher{}, &SynthEvent{Type: "step/screenshot_ref"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { @@ -305,7 +300,6 @@ func TestEnrichSynthEvent(t *testing.T) { }, { "step/screenshot_block", - &journeyEnricher{}, &SynthEvent{Type: "screenshot/block", Id: "my_id"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { @@ -316,7 +310,6 @@ func TestEnrichSynthEvent(t *testing.T) { }, { "journey/network_info", - &journeyEnricher{}, &SynthEvent{Type: "journey/network_info"}, false, func(t *testing.T, e *beat.Event, je *journeyEnricher) { @@ -327,11 +320,12 @@ func TestEnrichSynthEvent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + je := newJourneyEnricher(newStreamEnricher(stdfields.StdMonitorFields{})) e := &beat.Event{} - if err := tt.je.enrichSynthEvent(e, tt.se); (err == nil && tt.wantErr) || (err != nil && !tt.wantErr) { + if err := je.enrichSynthEvent(e, tt.se); (err == nil && tt.wantErr) || (err != nil && !tt.wantErr) { t.Errorf("journeyEnricher.enrichSynthEvent() error = %v, wantErr %v", err, tt.wantErr) } - tt.check(t, e, tt.je) + tt.check(t, e, je) }) } } @@ -520,6 +514,7 @@ func TestCreateSummaryEvent(t *testing.T) { end: baseTime.Add(10 * time.Microsecond), stepCount: 0, journeyComplete: false, + streamEnricher: newStreamEnricher(stdfields.StdMonitorFields{}), }, expected: mapstr.M{ "monitor.duration.us": int64(10), @@ -536,6 +531,7 @@ func TestCreateSummaryEvent(t *testing.T) { end: time.Now().Add(10 * time.Microsecond), journeyComplete: false, errorCount: 1, + streamEnricher: newStreamEnricher(stdfields.StdMonitorFields{}), }, expected: mapstr.M{ "summary": mapstr.M{ @@ -559,20 +555,22 @@ func TestCreateSummaryEvent(t *testing.T) { } else { require.NoError(t, err) } - //nolint:errcheck // There are no new changes to this line but // linter has been activated in the meantime. We'll cleanup separately. - mapstr.MergeFields(tt.expected, mapstr.M{ + err = mapstr.MergeFields(tt.expected, mapstr.M{ "monitor": monitorField, "url": mapstr.M{}, "event.type": "heartbeat/summary", "synthetics.type": "heartbeat/summary", "synthetics.journey": testJourney, }, true) + require.NoError(t, err) testslike.Test(t, lookslike.Strict(lookslike.MustCompile(tt.expected)), e.Fields) }) } } func makeTestJourneyEnricher(sFields stdfields.StdMonitorFields) *journeyEnricher { - return &journeyEnricher{streamEnricher: newStreamEnricher(sFields)} + return &journeyEnricher{ + streamEnricher: newStreamEnricher(sFields), + } } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go index 77d44846a4d..f37b966322f 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go index 2980a6da8ed..059423945e2 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go index f52788cf042..8ba0115ed28 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec @@ -20,12 +22,13 @@ import ( "sync" "time" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/beats/v7/heartbeat/ecserr" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" ) const debugSelector = "synthexec" @@ -41,6 +44,8 @@ type FilterJourneyConfig struct { // where these are unsupported var platformCmdMutate func(*exec.Cmd) = func(*exec.Cmd) {} +var SynthexecTimeout struct{} + // ProjectJob will run a single journey by name from the given project. func ProjectJob(ctx context.Context, projectPath string, params mapstr.M, filterJourneys FilterJourneyConfig, fields stdfields.StdMonitorFields, extraArgs ...string) (jobs.Job, error) { // Run the command in the given projectPath, use '.' as the first arg since the command runs @@ -87,12 +92,18 @@ func InlineJourneyJob(ctx context.Context, script string, params mapstr.M, field // Here, we adapt one to the other, where each recursive job pulls another item off the chan until none are left. func startCmdJob(ctx context.Context, newCmd func() *exec.Cmd, stdinStr *string, params mapstr.M, filterJourneys FilterJourneyConfig, sFields stdfields.StdMonitorFields) jobs.Job { return func(event *beat.Event) ([]jobs.Job, error) { + senr := newStreamEnricher(sFields) mpx, err := runCmd(ctx, newCmd(), stdinStr, params, filterJourneys) if err != nil { + err := senr.enrich(event, &SynthEvent{ + Type: "cmd/could_not_start", + Error: ECSErrToSynthError(ecserr.NewSyntheticsCmdCouldNotStartErr(err)), + }) return nil, err } - senr := newStreamEnricher(sFields) - return []jobs.Job{readResultsJob(ctx, mpx.SynthEvents(), senr.enrich)}, nil + // We don't just return the readResultsJob, otherwise we'd just send an empty event, execute it right away + // then it'll keep executing itself until we're truly done + return readResultsJob(ctx, mpx.SynthEvents(), senr.enrich)(event) } } @@ -241,9 +252,17 @@ func runCmd( return nil, err } - // Kill the process if the context ends + // Get timeout from parent ctx + timeout, _ := ctx.Value(SynthexecTimeout).(time.Duration) + ctx, cancel := context.WithTimeout(ctx, timeout) go func() { <-ctx.Done() + + // ProcessState can be null if it hasn't reported back yet + if cmd.ProcessState != nil && cmd.ProcessState.Exited() { + return + } + err := cmd.Process.Kill() if err != nil { logp.Warn("could not kill synthetics process: %s", err) @@ -253,13 +272,21 @@ func runCmd( // Close mpx after the process is done and all events have been sent / consumed go func() { err := <-cmdDone - jsonWriter.Close() + _ = jsonWriter.Close() logp.Info("Command has completed(%d): %s", cmd.ProcessState.ExitCode(), loggableCmd.String()) var cmdError *SynthError = nil if err != nil { - cmdError = ECSErrToSynthError(ecserr.NewBadCmdStatusErr(cmd.ProcessState.ExitCode(), loggableCmd.String())) + // err could be generic or it could have been killed by context timeout, log and check context + // to decide which error to stream logp.Warn("Error executing command '%s' (%d): %s", loggableCmd.String(), cmd.ProcessState.ExitCode(), err) + + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + timeout, _ := ctx.Value(SynthexecTimeout).(time.Duration) + cmdError = ECSErrToSynthError(ecserr.NewCmdTimeoutStatusErr(timeout, loggableCmd.String())) + } else { + cmdError = ECSErrToSynthError(ecserr.NewBadCmdStatusErr(cmd.ProcessState.ExitCode(), loggableCmd.String())) + } } mpx.writeSynthEvent(&SynthEvent{ @@ -270,6 +297,7 @@ func runCmd( wg.Wait() mpx.Close() + cancel() }() return mpx, nil @@ -278,7 +306,9 @@ func runCmd( // scanToSynthEvents takes a reader, a transform function, and a callback, and processes // each scanned line via the reader before invoking it with the callback. func scanToSynthEvents(rdr io.ReadCloser, transform func(bytes []byte, text string) (*SynthEvent, error), cb func(*SynthEvent)) error { - defer rdr.Close() + defer func() { + _ = rdr.Close() + }() scanner := bufio.NewScanner(rdr) buf := make([]byte, 1024*10) // 10KiB initial buffer scanner.Buffer(buf, 1024*1024*10) // Max 10MiB Buffer diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go index cc6d88f53f0..65843d72737 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_linux.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go index 1e3cdd2aae3..3ff620e67be 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go @@ -2,11 +2,15 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin + package synthexec import ( "context" "fmt" + "os" "os/exec" "path" "path/filepath" @@ -95,11 +99,23 @@ func TestJsonToSynthEvent(t *testing.T) { } } +func goCmd(args ...string) *exec.Cmd { + goBinary := "go" // relative by default + // GET the GOROOT if defined, this helps in scenarios where + // GOROOT is defined, but GOROOT/bin is not in the path + // This can happen when targeting WSL from intellij running on windows + goRoot := os.Getenv("GOROOT") + if goRoot != "" { + goBinary = filepath.Join(goRoot, "bin", "go") + } + return exec.Command(goBinary, args...) +} + func TestRunCmd(t *testing.T) { - cmd := exec.Command("go", "run", "./main.go") + cmd := goCmd("run", "./main.go") stdinStr := "MY_STDIN" - synthEvents := runAndCollect(t, cmd, stdinStr) + synthEvents := runAndCollect(t, cmd, stdinStr, 15*time.Minute) t.Run("has echo'd stdin to stdout", func(t *testing.T) { stdoutEvents := eventsWithType(Stdout, synthEvents) @@ -132,8 +148,8 @@ func TestRunCmd(t *testing.T) { } func TestRunBadExitCodeCmd(t *testing.T) { - cmd := exec.Command("go", "run", "./main.go", "exit") - synthEvents := runAndCollect(t, cmd, "") + cmd := goCmd("run", "./main.go", "exit") + synthEvents := runAndCollect(t, cmd, "", 15*time.Minute) // go run outputs "exit status 123" to stderr so we have two messages require.Len(t, synthEvents, 2) @@ -149,11 +165,27 @@ func TestRunBadExitCodeCmd(t *testing.T) { }) } -func runAndCollect(t *testing.T, cmd *exec.Cmd, stdinStr string) []*SynthEvent { - _, filename, _, _ := runtime.Caller(0) - cmd.Dir = path.Join(filepath.Dir(filename), "testcmd") +func TestRunTimeoutExitCodeCmd(t *testing.T) { + cmd := goCmd("run", "./main.go") + synthEvents := runAndCollect(t, cmd, "", 0*time.Second) + + // go run should not produce any additional stderr output in this case + require.Len(t, synthEvents, 1) + + t.Run("has a cmd status event", func(t *testing.T) { + stdoutEvents := eventsWithType(CmdStatus, synthEvents) + require.Len(t, stdoutEvents, 1) + require.Equal(t, synthEvents[0].Error.Code, "CMD_TIMEOUT") + }) +} + +func runAndCollect(t *testing.T, cmd *exec.Cmd, stdinStr string, cmdTimeout time.Duration) []*SynthEvent { + cwd, err := os.Getwd() + require.NoError(t, err) + cmd.Dir = filepath.Join(cwd, "testcmd") + ctx := context.WithValue(context.TODO(), SynthexecTimeout, cmdTimeout) - mpx, err := runCmd(context.TODO(), cmd, &stdinStr, nil, FilterJourneyConfig{}) + mpx, err := runCmd(ctx, cmd, &stdinStr, nil, FilterJourneyConfig{}) require.NoError(t, err) var synthEvents []*SynthEvent diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 2619a6292d9..75ee7bed641 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go index ae88310fb3b..6a1e416e2be 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go @@ -1,6 +1,8 @@ // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin package synthexec diff --git a/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go b/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go index 33c85abe665..4627e4edc20 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build linux || darwin +// +build linux darwin + package main import ( @@ -12,6 +15,8 @@ import ( ) func main() { + //Sleep first to test timeout feature async + time.Sleep(time.Millisecond * 500) // For sending JSON results pipe := os.NewFile(3, "pipe") diff --git a/x-pack/heartbeat/scenarios/basics_test.go b/x-pack/heartbeat/scenarios/basics_test.go new file mode 100644 index 00000000000..36e5abf7776 --- /dev/null +++ b/x-pack/heartbeat/scenarios/basics_test.go @@ -0,0 +1,80 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package scenarios + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/go-lookslike" + "github.com/elastic/go-lookslike/isdef" + "github.com/elastic/go-lookslike/testslike" + + "github.com/elastic/beats/v7/heartbeat/hbtestllext" + _ "github.com/elastic/beats/v7/heartbeat/monitors/active/http" + _ "github.com/elastic/beats/v7/heartbeat/monitors/active/icmp" + _ "github.com/elastic/beats/v7/heartbeat/monitors/active/tcp" + _ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser" +) + +func TestSimpleScenariosBasicFields(t *testing.T) { + Scenarios.RunAll(t, func(mtr *MonitorTestRun, err error) { + require.GreaterOrEqual(t, len(mtr.Events()), 1) + lastCg := "" + for i, e := range mtr.Events() { + testslike.Test(t, lookslike.MustCompile(map[string]interface{}{ + "monitor": map[string]interface{}{ + "id": mtr.StdFields.ID, + "name": mtr.StdFields.Name, + "type": mtr.StdFields.Type, + "check_group": isdef.IsString, + }, + }), e.Fields) + + // Ensure that all check groups are equal and don't change + cg, err := e.GetValue("monitor.check_group") + require.NoError(t, err) + cgStr := cg.(string) + if i == 0 { + lastCg = cgStr + } else { + require.Equal(t, lastCg, cgStr) + } + } + }) +} + +func TestLightweightUrls(t *testing.T) { + Scenarios.RunTag(t, "lightweight", func(mtr *MonitorTestRun, err error) { + for _, e := range mtr.Events() { + testslike.Test(t, lookslike.MustCompile(map[string]interface{}{ + "url": map[string]interface{}{ + "full": isdef.IsNonEmptyString, + "domain": isdef.IsNonEmptyString, + "scheme": mtr.StdFields.Type, + }, + }), e.Fields) + } + }) +} + +func TestLightweightSummaries(t *testing.T) { + Scenarios.RunTag(t, "lightweight", func(mtr *MonitorTestRun, err error) { + all := mtr.Events() + lastEvent, firstEvents := all[len(all)-1], all[:len(all)-1] + testslike.Test(t, lookslike.MustCompile(map[string]interface{}{ + "summary": map[string]interface{}{ + "up": hbtestllext.IsUint16, + "down": hbtestllext.IsUint16, + }, + }), lastEvent.Fields) + + for _, e := range firstEvents { + summary, _ := e.GetValue("summary") + require.Nil(t, summary) + } + }) +} diff --git a/x-pack/heartbeat/scenarios/framework.go b/x-pack/heartbeat/scenarios/framework.go new file mode 100644 index 00000000000..4438994a684 --- /dev/null +++ b/x-pack/heartbeat/scenarios/framework.go @@ -0,0 +1,180 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package scenarios + +import ( + "os" + "sync" + "testing" + "time" + + "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/monitoring" + + "github.com/elastic/beats/v7/heartbeat/monitors" + "github.com/elastic/beats/v7/heartbeat/monitors/plugin" + "github.com/elastic/beats/v7/heartbeat/scheduler" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/publisher/pipeline" + beatversion "github.com/elastic/beats/v7/libbeat/version" +) + +type ScenarioRun func() (config mapstr.M, close func(), err error) + +type Scenario struct { + Name string + Type string + Runner ScenarioRun + Tags []string +} + +func (s Scenario) Run(t *testing.T, callback func(mtr *MonitorTestRun, err error)) { + cfgMap, rClose, err := s.Runner() + defer rClose() + if err != nil { + callback(nil, err) + return + } + + t.Run(s.Name, func(t *testing.T) { + t.Parallel() + mtr, err := runMonitorOnce(t, cfgMap) + mtr.Wait() + callback(mtr, err) + mtr.Close() + }) + +} + +type ScenarioDB struct { + All []Scenario + ByTag map[string][]Scenario + initOnce *sync.Once +} + +func (sdb *ScenarioDB) Init() { + var prunedList []Scenario + browserCapable := os.Getenv("ELASTIC_SYNTHETICS_CAPABLE") == "true" + icmpCapable := os.Getenv("ELASTIC_ICMP_CAPABLE") == "true" + sdb.initOnce.Do(func() { + for _, s := range sdb.All { + if s.Type == "browser" && !browserCapable { + continue + } + if s.Type == "icmp" && !icmpCapable { + continue + } + prunedList = append(prunedList, s) + + for _, t := range s.Tags { + sdb.ByTag[t] = append(sdb.ByTag[t], s) + } + } + }) + sdb.All = prunedList +} + +func (sdb *ScenarioDB) RunAll(t *testing.T, callback func(*MonitorTestRun, error)) { + sdb.Init() + for _, s := range sdb.All { + s.Run(t, callback) + } +} + +func (sdb *ScenarioDB) RunTag(t *testing.T, tagName string, callback func(*MonitorTestRun, error)) { + sdb.Init() + if len(sdb.ByTag[tagName]) < 1 { + require.Failf(t, "no scenarios have tags matching %s", tagName) + } + for _, s := range sdb.ByTag[tagName] { + s.Run(t, callback) + } +} + +type MonitorTestRun struct { + StdFields stdfields.StdMonitorFields + Config mapstr.M + Monitor *monitors.Monitor + Events func() []*beat.Event + Wait func() + Close func() +} + +func runMonitorOnce(t *testing.T, monitorConfig mapstr.M) (mtr *MonitorTestRun, err error) { + mtr = &MonitorTestRun{ + Config: monitorConfig, + StdFields: stdfields.StdMonitorFields{}, + } + + // make a pipeline + pipe := &monitors.MockPipeline{} + // pass it to the factory + f, sched, closeFactory := setupFactoryAndSched() + conf, err := config.NewConfigFrom(monitorConfig) + require.NoError(t, err) + err = conf.Unpack(&mtr.StdFields) + require.NoError(t, err) + + mIface, err := f.Create(pipe, conf) + require.NoError(t, err) + mtr.Monitor = mIface.(*monitors.Monitor) + require.NotNil(t, mtr.Monitor, "could not convert to monitor %v", mIface) + mtr.Events = pipe.PublishedEvents + + // start the monitor + mtr.Monitor.Start() + mtr.Wait = func() { + // wait for the monitor to stop + sched.WaitForRunOnce() + // stop the monitor itself + mtr.Monitor.Stop() + closeFactory() + } + mtr.Close = closeFactory + return mtr, err +} + +func setupFactoryAndSched() (factory *monitors.RunnerFactory, sched *scheduler.Scheduler, close func()) { + id, _ := uuid.NewV4() + eid, _ := uuid.NewV4() + info := beat.Info{ + Beat: "heartbeat", + IndexPrefix: "heartbeat", + Version: beatversion.GetDefaultVersion(), + ElasticLicensed: true, + Name: "heartbeat", + Hostname: "localhost", + ID: id, + EphemeralID: eid, + FirstStart: time.Now(), + StartTime: time.Now(), + Monitoring: struct { + DefaultUsername string + }{ + DefaultUsername: "test", + }, + } + + sched = scheduler.Create( + 1, + monitoring.NewRegistry(), + time.Local, + nil, + true, + ) + + return monitors.NewFactory(info, sched.Add, plugin.GlobalPluginsReg, func(pipeline beat.Pipeline) (pipeline.ISyncClient, error) { + c, _ := pipeline.Connect() + return monitors.SyncPipelineClientAdaptor{C: c}, nil + }), + sched, + sched.Stop +} diff --git a/x-pack/heartbeat/scenarios/scenarios.go b/x-pack/heartbeat/scenarios/scenarios.go new file mode 100644 index 00000000000..f4474643cc4 --- /dev/null +++ b/x-pack/heartbeat/scenarios/scenarios.go @@ -0,0 +1,99 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package scenarios + +import ( + "fmt" + "net/http/httptest" + "net/url" + "os" + "sync" + + "github.com/elastic/elastic-agent-libs/mapstr" + + "github.com/elastic/beats/v7/heartbeat/hbtest" +) + +var Scenarios = &ScenarioDB{ + initOnce: &sync.Once{}, + ByTag: map[string][]Scenario{}, + All: []Scenario{ + { + Name: "http-simple", + Type: "http", + Tags: []string{"lightweight", "http"}, + Runner: func() (config mapstr.M, close func(), err error) { + server := httptest.NewServer(hbtest.HelloWorldHandler(200)) + config = mapstr.M{ + "id": "http-test-id", + "name": "http-test-name", + "type": "http", + "schedule": "@every 1m", + "urls": []string{server.URL}, + } + return config, server.Close, nil + }, + }, + { + Name: "tcp-simple", + Type: "tcp", + Tags: []string{"lightweight", "tcp"}, + Runner: func() (config mapstr.M, close func(), err error) { + server := httptest.NewServer(hbtest.HelloWorldHandler(200)) + parsedUrl, err := url.Parse(server.URL) + if err != nil { + panic(fmt.Sprintf("URL %s should always be parsable: %s", server.URL, err)) + } + config = mapstr.M{ + "id": "tcp-test-id", + "name": "tcp-test-name", + "type": "tcp", + "schedule": "@every 1m", + "hosts": []string{fmt.Sprintf("%s:%s", parsedUrl.Host, parsedUrl.Port())}, + } + return config, server.Close, nil + }, + }, + { + Name: "simple-icmp", + Type: "icmp", + Tags: []string{"icmp"}, + Runner: func() (config mapstr.M, close func(), err error) { + return mapstr.M{ + "id": "icmp-test-id", + "name": "icmp-test-name", + "type": "icmp", + "schedule": "@every 1m", + "hosts": []string{"127.0.0.1"}, + }, func() {}, nil + }, + }, + { + Name: "simple-browser", + Type: "browser", + Tags: []string{"browser", "browser-inline"}, + Runner: func() (config mapstr.M, close func(), err error) { + err = os.Setenv("ELASTIC_SYNTHETICS_CAPABLE", "true") + if err != nil { + return nil, nil, err + } + server := httptest.NewServer(hbtest.HelloWorldHandler(200)) + config = mapstr.M{ + "id": "browser-test-id", + "name": "browser-test-name", + "type": "browser", + "schedule": "@every 1m", + "hosts": []string{"127.0.0.1"}, + "source": mapstr.M{ + "inline": mapstr.M{ + "script": fmt.Sprintf("step('load server', async () => {await page.goto('%s')})", server.URL), + }, + }, + } + return config, server.Close, nil + }, + }, + }, +} diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 34ac4a18469..fb00273853a 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,12 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/libbeat check; - make -C x-pack/libbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 0467c02df2c..f45989ac964 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -185,7 +185,7 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName beatsConfig.EndpointResolverWithOptions = awssdk.EndpointResolverWithOptionsFunc( func(service, region string, options ...interface{}) (awssdk.Endpoint, error) { - return awssdk.Endpoint{URL: eurl}, nil + return awssdk.Endpoint{URL: eurl, SigningRegion: region}, nil }) } return beatsConfig diff --git a/x-pack/libbeat/common/aws/credentials_test.go b/x-pack/libbeat/common/aws/credentials_test.go index 2a7d7a31ea0..aa1f757317c 100644 --- a/x-pack/libbeat/common/aws/credentials_test.go +++ b/x-pack/libbeat/common/aws/credentials_test.go @@ -109,7 +109,7 @@ func TestEnrichAWSConfigWithEndpoint(t *testing.T) { func getEndpointResolverWithOptionsFunc(e string) awssdk.EndpointResolverWithOptionsFunc { return func(service, region string, options ...interface{}) (awssdk.Endpoint, error) { - return awssdk.Endpoint{URL: e}, nil + return awssdk.Endpoint{URL: e, SigningRegion: region}, nil } } diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index fd3e5fb347b..a61132526c8 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -13,14 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/metricbeat check; - make -C x-pack/metricbeat update; - make -C metricbeat check; - make -C metricbeat update; - make check-no-changes; - stage: checks unitTest: mage: "mage build unitTest" stage: mandatory diff --git a/x-pack/metricbeat/module/aws/billing/billing.go b/x-pack/metricbeat/module/aws/billing/billing.go index 0fc8b9c331d..796c5eccef0 100644 --- a/x-pack/metricbeat/module/aws/billing/billing.go +++ b/x-pack/metricbeat/module/aws/billing/billing.go @@ -124,7 +124,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { startDate, endDate := getStartDateEndDate(m.Period) // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency) // get cost metrics from cost explorer awsBeatsConfig := m.MetricSet.AwsConfig.Copy() @@ -444,8 +444,8 @@ func (m *MetricSet) getAccountName(svc *organizations.Client) map[string]string for paginator.HasMorePages() { page, err := paginator.NextPage(context.Background()) if err != nil { - //TODO continue or return with error? Probably is return but... - continue + m.Logger().Warnf("an error occurred while listing account: %s", err.Error()) + return accounts } for _, a := range page.Accounts { accounts[*a.Id] = *a.Name diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 1e98477f6af..ccee583a460 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -126,7 +126,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.Period, m.Latency) m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // Check statistic method in config diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 12a19003fe7..f696d110e7d 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1392,7 +1392,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { Value: "test-ec2", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1432,7 +1432,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { } resourceTypeTagFilters := map[string][]aws.Tag{} - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1478,7 +1478,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) assert.Equal(t, 1, len(events)) @@ -1630,7 +1630,7 @@ func TestCreateEventsTimestamp(t *testing.T) { } resourceTypeTagFilters := map[string][]aws.Tag{} - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) cloudwatchMock := &MockCloudWatchClientWithoutDim{} resGroupTaggingClientMock := &MockResourceGroupsTaggingClient{} @@ -1644,6 +1644,6 @@ func TestGetStartTimeEndTime(t *testing.T) { m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} m.MetricSet = &aws.MetricSet{Period: 5 * time.Minute} m.logger = logp.NewLogger("test") - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) assert.Equal(t, 5*time.Minute, endTime.Sub(startTime)) } diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index b84019f8b6b..713690883f3 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded zlib format compressed contents of module/aws. func AssetAws() string { - return "eJzsfVtzI7fx77s/BSov3k1pFXvXTp3yw6nSzbFOtFpZ1MZ+Y8CZJokIA8wCGGnpyoc/hQYwg7nxOkPJ//rvS2KRBH7daDS6G43ud+QRVj8R+qy/IcQww+En8pez3yZ/+YaQFHSiWG6YFD+R//sNIYT8mz7rf5NMpgUHkkjOITGanP02IZkUzEjFxIJkYBRLNJkrmeFnF1wW6TM1yfL0G0IUcKAafiIL+g0hcwY81T/h6O+IoBkENPafWeX2i0oWuf9LB6j6IPFAhi706V/LP4fx5Ow/kJjoz+4PU/fpI6yepUq7P55mNM+ZWPjv/uWvf4m+14nN/XugCzsweaK8AJJTpjx/6LMmCrQsVAL6tEWB/nA6K5JHMKf2v1uUtLGuwXBLMyByTiiZfCB+1NaEKctAaCbFK2HcRxSmGFYL8rd/PfUid/rX079+uyPqVBYzDmOA1sQsqSEKTKEEpG69q71Azu6uyZcC1KpNEmfiEdIpTRJZCNOiKN4QpEP+46FYWvtzv+RsoMn+u74khYaUGElYCsKw+cpDJR7qaSeGhuweiMLJsSKUM6q3BxTAzBjnTCw2MnUNin/7Mf5NEikMZcIuNRDQhmXUQEqSJVUL0GQuFVnJQqEa9IgIEw2NGP6VmnEGhm65vFdhzgs3ZSebuazR26LuI/3KsiLrIcBjX7O+F4VSIJLVvmt81Zo38SOSQrCeSSegnlgCtwfIlh8CB0RS7SpmfczohnGWSWXYH5BeSG06gTQFq29J41Fp1tj49SFbSquTvBIaSaQ2fWOGKS2neyfsZuamGVtDhrnOOYj0NbLMAzsaw2rz9bLrVqqMcsvXz5ou4KwL1wszroJICovxGMzrmbOfj5/F7LUKXgntaKLXmLGfaZa1vxZUGGa6NfzLMQ1X/YvHdhSm1WfsZZo2VJlpSs3+Z5MdgdgR8GRS1qSEJ+tf2fPYLpnunBlEetC8VyLdY1YUgWkKcyaYHWcwOXmEpsxtoqZF0cMSiDbomnqDPFegQRhNKDplllJKdA4JmzNIO3FGTuUq7xLMoSBZE8SOZT21NpA6v2ermpNG+l0estlZIzv4Pi2C2la6VbEEvuZcKlAOL5mtKidYtwzzpDSKD7LNq2Ea5nkWe2XPoIDoRNE8eGZlpOI39M6elyxZVgN0xDfselmSUjafg7L/YenQOU3qtmI94BH+rTPqy3GGc5qsxJXDRrL+vAThvNCI/4TmrCM0sBI0k+nsoNUJgxxpbewPL3HKy/NDXS0/9mCqbVIkCWg9L/g9fClAmxtqrM9zSp+a3hrZ8WBsrz/xMkCfQNkjjLu5rJbRJQ6iHBBtHerANutqn2X0DymqP02MApo1WeGBFF6vxWJmWAYkB8Vkero7QzL6dTSGBHfvNTLkk+BMwLVI4esdqASEoQu4U3KhQOtRxSQvp7MMSWSWc7C/cfqCEgHPZMHljHKiIZEipWpFmAVKmCYzsATTNHWhGUoMnXHop/NOySemmRSQ/qaYgQua04SZ1WfBzLh0iiKbgbI05hUG8mxBkMSjQCtPeysBKcHoUw/9W1F5DzR9aSIV0HRwGi+k0EV2bAKDUqsI7SIu8diIfALVvx1POqfRkqxkQRIqiFE0eSRL+UyyIlna2TDEF/PWLJUsFsu8MHY7FBrWbPJ+luki62VZR0hvB4bpIvuTcunI+qEtWZ264c/HtNFl68/Ep3vIOUuopeyYNhhwmutA+QzMM9izVZAiTzHuzAxkhOY5UDQgmECOlTaHRpvD6uzOmaQA61dawpxGPyFUpM7Cbo9MhTRLUOUv/GRe/284vzv4dwyT7X8M/x4UFZomlu4LKeacJWY0ATzzwqfAuvqeS+84PEFk7aYFWMPNVLgot5sXoemS14kU7qKmK65GquGkY4amGeB0ejdWjKSrpKH8tbLhzF239ZmMhnH2B+63oyiqujewyYgsEB2k1v+29HZeDa8ntn5gvRpqO8+0ncmdrLSB7EopqcY8h3d0XZ1iW4AA1Y4eu39UkF8eHu7Ij999R7ShprAHegoHOLgXUqTM7auLJSSPP1PGrag75CMyp7Ln5jglocZAljtu5aDmUmV2Xwd0bunXbNg7ECkTi+gkvEApOAYJeBq5Q88vI1WAiA0IS1D7KOscdVYY9/MlfQIipCErMGRmVVw02IGWAk0flkoaw+HqCcRoi3zfJf1IHHxNAO1D6NdknUMO5CIH8scW8505EFnMnGXMdEezpCC0zFgjb7S1v6musUQ4Frzt5wHq99cpB3UdP6Yg+GPvI/1qd4VeazIfpiqCwbw+PoJcsd7VDFxW1Wxl17L3PHOjM+2khaQSNCoNmud85dTOuxQyNJotl7RlUzeT1mnWik0PdpQba6K9YoZVEuFI7XZlGzFTOY85TX6Wqs08U7E6obmOc5V6zE6aBiPAA95CXJGeQq9R4V3r8Zs7HY+5IJ222OteEQd51CV51Qvx8rrkI/0aeRkov31+1ZgBjMP8qSVbLKGVv+T+tcZqyP4GOd+Fcb0+2stwrimG3UyLf7Jmj+7JtTIHZxbbTrtfksNMH/F+/Op8cli6wtAX4/+SvMhwY56vrDY73OkPQS/N/kDBAZos3f6QufV3mRSxF+uj0Ggi5saavE8ISVs3kSbLcK15y4yS72bUKjgmtKEigRPyvLTrY6KIQiO9J/y5Iwi+yWF2rMGtNypv3Db4UzLHys2nfAjOWIVjMErYsANLvmgM/bYg2i8alq05saN1HA9rYxEPBPtrAQXcgFiY5UB4G1y1h3tT7sog1jNlBiVQWpPCJySgZB1A0kPp8VbpFQPRVj+orv/2KV6HHJQ/Usib6093k7ckBc6eQIGDXq6l/bB2ys2df+1jeFfnE7/5Tslnu8+emVnGeQZugMnkstyjUvDVJrbEN9KjiKjP016z8Jq8EVV2t5Hk/Y9//2fDMHpbXSeul4JheHNeKG3OKbd6bABuVJj+gTFXTu4KlUsNCOnNIn//9oRUAko+5YZlyI1fLi/JG22+f+supC4kD39Lvn9bJ8bRm4Ld+nPLT9xUdCYx0tclpYmC1Bqdb6ykWRAEn8WUMGqfa/M9QsCJFWSUieiibWYZ1npo2C1yeBmDwUG7YOtCQfurQ7fjtJUTZ/xQzlv63DkuA6kXC8CFuo5MVWs3DUnWdcqPQdBajC4PTUi/fqpNsTOSi1nGjInv/ksbPXl/mI2evD+mjX7x/jAbPcmLU+T0ad5KDHfE64RySKdzLmnzC1vkFtc1CeVcJngHf3XxHuWuMBCHBqgC/8bPcOtUkUJDuB8NxmL3eztLiFNCU3z000nLpgePPfnRpQxe3H0uNV25sWJseBDbbxWR47sJ78wdHqMgBopvjGPgjtGiwryk2vqsqoCUaGb/wgx5pppwWgg03FGnU2WayTIxMbpQOS/09AhE+anqFOHlFF5KVSpPkEJg5CjyNZyKsD+7uPt8gSP409u/wmea/AFKbkupnrp3oN3vqQ8mFWnpJNjuFSENySlLSSqfhSW5vd7OGnBqxSwLq0CTAq1FmpbXmI6EnlfaYJ6lejxl4jSn9tDe7zFxN6VNLe9nIAoSYE9W9ASeXB4EYcKAmtMEdGvrMRFKT1hjpssp7KdomoOaakhG0IBt2iIzH3W5tbq2JnM9RbIwR1yk3dHvsUgRSf9TVomJ09nKbP8o35noP5GuH+2xfDjM0XYYznaUlXN0Reu2O4mbRfHlF+5ou+4FV26oHZcy/cjkqfUGjrdyuGphk1Fv5lsqyvXQRiqo4qNPlHG8WTBy33VrETrSup1XZEXLtTeFa4lB3+1Fli1OazrKukWkjrpwgbBo7fakcbMYNmsUrV24rRanClQ0wzPH3mJI29qV2p3Gi17qhthpu8R2OoVzzOVsx6WOu/HGXc4WdYfvvn1W06XmniZLSB6nLr11IFLvIZfKaOtZYwpoDemSapJTjcke0izrH4Z0YYvJP6MAojERuv6Zjx1zqg3JmCjM9kRO3XhHpnUMQsI8L0BK94ptS0x5aCRSrdMk1rxbQPPdze4hOql8jbLNJ1b5KcvoAk6HLItngV1fhps7HL8sS+dCa7vgqyLBp3YNBqwBcS1SlmCSeJCEFIxLf4/Cz0wTEFYX9SjUEmiu2BM1cJoKPR22wh8GlN3o5PJ24uqzefa2PIQtUbJmFoqXxOafd4B2fff0A6FpqkBrQrWWCcOYN97q7YW1mHGWjMVQHLzFzy2l0kMbkIuBcR7HlVUuLCHXd+UnbyyD35KZLNwBug9LcQudJjLt5ubeigjHbfLwxGXCf//3dzNmSCE0WwiMSOMkWyEdft07kZI3uXuwQv5LVCGE+396WRjDxOIdRpn/SwyojAmU6f9aiwULAoX/C+nbDRSZpTVwnaNjVfVYR4GfB82tcCx0XPjxwyrXAD9m0Zqrm+56NS+WlHdOk0cQ6YUUwlndAz1gqy9lUg4fs1VIExVl4SsC2tAZZ3ppjU3/ChMNFElT4m+kVGlnKlgwbTC7JsjmmhzhXx4e7i5kClNP8fT9778PTCW+onv/++9Egc6l0ODe0YXHd5i0eiDoD+OA/jAq6B/GAf3DqKB/HAf0j6OAvro5H5PLCWdWh4FVDQha11G39uiWkEfksQb1BGoQyP6t2TAPP5sJkj4PsoqlINxKW2a07yUumkpPlK95kZwzzuUTqOGgt/Nmwzu8UquXT+9nkNBCu6xgXSgssAnugt6q+zUyApSb5eoXGZh+6LuXOtOXbvhqg8W7Do18rDqypXRMLGVxEu0QYHvZ/AYFnFu0AtTbprS8ebiIPy3zDIJVqGQR0m1piw/9NH4WIy9JIYZdlOHKvVSrgflpvjbJCWEiZLSdOLMQs3vtV9oGCxqApnq779jfoepJIQzjrYCNMi7ooKG0fPwBsgSaglpzQpQl2M9uzs8Sw56gsvTcQg7Doqqqes3o87lgxIplLKcUoTjGucNFB0+wbeuV7K1/ZL9P1QLMluSH9Oebi89DpT13UV0H2Xjz9ebm4vPb+OXcWV4WFiA39pfnG2U7pukWno+3ngKeWwsZW+zHW807Ja3TAIM9JOoj2V9sh+m2X7SyHnb11UMd1fpQR/RZI3JfnfvardPGsHRegTa7wLEfbia3sJCG0dJdH8M0fbiZ1IjECuCx9eydApS4lKXozZfqgFCiQWssLRrCpnWCfREmihOhmb7eaZj+zL5COr33R990DJrndop35elKWxGLKlqxAew9pExBYkaBqfzggwD8rPj0hmXMTK+wcgakR8ScyIKn4ltTf/wVOw6f72/CNVW5LpiEbkXLmT/WoeB27yg7qCD/559bup8ffv99FFqjkIoj2mJ1PihSLRVbYPy1Rxls7/CPB7/H7R8S/49j4u+JAQyK/7vvRsT/3XcjAn8/JvD3IwL/MCbwDyMC/2FM4D8MCfz67unvDQN7DHuqw7RuGwn4WtwCWg93xAidHb4Kv5QZybtFEDvctDFY+uIO2msTmx+QoPXyc+/DlWMs0KYLsM5QaZ2UJVZ7cvUXmNEdhXqioV82hl0tyk78LzhcPVFeuOS6ocEVfLO4LNgTuPJ3LjynrNr0BSs8MVSQpSzWbPERokt7xZR2iZKOHNT16iJ6GSqFZilGPH249wVDzuvQleHodkDHJ6ocGsyphjliIOfWTfpKgzg/c/k8ZAhzTQBnzuWzJm/qlydv2+fjpvOuAXz6cHE3Pnh7wo9GwM3kCATcTEYj4PPlEVbg8+VwK/BnPDeOEIdsct/KzJKKVC/pY3BxfIlnfzkuKixV04AQwrBmiIs0hsvRtcZ6pYrGMtN7xGette4PLB8N26oQd0wLbu7R3I7+PT00Ta/EyTghTCS8wGv1h4u7v13fbb6NrUMfbUE64Meiv65NA67Hn2JnxxT5/e2kaQ11F3dTp7um96BhyOB8O2FDgyFv7icPb+vP7d0DsPLyRG4J++rm/EUw75szZTE7YXpxVjv2OlY7tv+vRzSkR/TIBGh2WGVUP8axfCFXZu+fbtJOX+gF+4f+A8w9JFKlejpUesMujdDCw3fsEA1RONCzy/dXOiEZUF2ojU2/+gU6IvTaWC0j1dkCPjLOmU+tGpf0Rfl8Ah/AKcSCLzM5j8CRhHLuEzHpwsqWIXQ4bth/ZwvMirS/Cl2HE6hlwAfXA4dyBYFAtLB7chrYsRZU9CTeuC7VQLOt1uYYrdNc4hx99E/bIwLKZ7fDCpz/36P0cVGelI49pZdUpcNS5rvtHoWyqKNu15L5l9JD6YtrkciMicX4WrFVsiV+pJIXJuyiuhLYRJirhO+OC+88YB0UOwNKxF3heYg7vPyvmKObuRMk+zj8CbI9Joe8csNnx0Nwqvz6Ec7XllPWy5pChxT/iriqYPK+e6ai9QXUeAchA6iBiqSg6o7XMTtSeK5zmHMrBjeNKol+QRtwJ1nVQwrrcYyOQHef1A5rfETEbSO3hx3RTR+5U0P6ov3UEOv0GN9ODvcryjmkkJ4gS8aUb18iemxzrB018AeXtaqxGkunKMfUp9TQUVgwKbXKMc3SSJcFZrwwH0KbyJcwzX0ihs9Kxrd1gnKMRBYKXpw1UXfBl+eOcWBCi+Rjs+UeaBp3LSqr2IcE5hEVa8WcVoDAhCUqK9dsZ/T20jkpZhbTDB7kxPqJ03tqYHQaIwNcE3Bl1l20geJNj3aocJTQXhW3iY6zmOy5whXQdGWHwaZK+Eai9msfUsbGyb7ngyJSEYZ93+NOjtE1LPI6qsBFOZfPJdNZJII7cHbsE7liathTcdGmJpw6l56p7q2SsD2J2I1zC2NyqO3RKFI7eMSjmz4XPDyHJROpNSH1+mSSw4gdIlTXWnqwdOwTsetmyMscF8dd9ONt3kgjwhOoVVhjv2pMu2pMeNUdb9lTco2fSmG3b6xTUVV+26ch+zmB7Ude/hDcwkLY7TDsjgDFw+0c/gks4zSbpfSgayo3xBEz9m5wwteVrXctnvz7q+ETlqwoaJeLNC9E9XAKd95XSArj3t6HxIvIh3Efu/eYIo3/ExdGgS649/TKoTc8O/SlkIYmklUM3B/bJdD0BowBNRjKn6UiVK9EslRSSOw0EYCeNKwwt05OOms99rFEQakQMTiWAk3fcYTqC4DMCm8xriNPGyZw7kvXQ231s3fFXjOlJeitaCy8nTqMgFXd13wZCmq6dpLOQaRldpDdRIGINdkc3rUZcy80SslQbCfpPao1tx7VxfFAcuHW0/fBzyjWxiv3aagG7w4z7YSl/yLZ/mkDay/KtNKrUmMNzuVSAqrqKlVpmEoQwHXs78X6WSjAN5XpSKh/9m0Qf5uQe1h07EaHsAI/A4u7lukWaPXfSqX41jcXCuCrVN5kTbpNZFx1Ujts7s1OK0SkqHWP3JueZOsGz9tQhKtHnkDhs3/7H5xRv0dcayY538RWkrInllbpZs3Gkj1kV53JdmVAbM0M+/RoG1tmwJUsi329PEVWglOq6ivkQkic9y4h075hXMejG2oW1MAzXR1kvlfD9JjwqNvRWH9GY916MoomjwRb0lkW3J49ED+GNcWpK/fvTotXl0mG0Z5r8bOSWWRPDSwUjUCP37cxn8owQGQf9Ut3BHqCbH0ZvCFJnQkn8P+6u9iA+VNhHuTYfC4b6/jerS3wPlq0PasR9oic9sXP1qLdidnVQ90zZ46P+163MvoxA7CHkm3gXlVx27GfGMdvLnZGjA7lnVTmjIdKK6McJE1hwGIwWEYonOaEBkM8l2qNER1a18piZGHwVplRVGjsvRhHOUMADwtzh4YfKfd/WXOeu6T1SyXzMdCHnPhUYXXvDo23EdrYZ0ira+TBp0gN+CjabWvMOyk3j3vks6TVAHKI0ySGPirHBz9RuovIjVHHNXpE6rVFs17HRm0dQKv0sCcXKj3acwvrzl9ODotiY9f50JV4n2bXobWcT+j7Zs3KrW2F7e7+e1r7O5jvXHP8WuP9shVUd3OFg/p4D0Wa93F27X/9v/26j92vO6WGzqiGaaQ5RiEnTNSoBFn30mvIZmWXq1OqRCeovVqe+Ddb997XJrc0gzdn97dvUQSAJkurETeDSjjV3bzaC9ZFrEBF1H8nNMenIiUZZFKtqvf3iCF88fJ8U0/GCD1LQRg2Z63GKkOQQO2yqne6yHPOIK0Wv5rV389WfwjPlgrBvhRgATh5L79hh92JRNegbDjyJv6eWdeSM6LuOT4/zVLag47pxyneXE1TyM2yE9vBPXRlYTBuZk/Q60+avFFA07+53oPhauQteaasLOCOV5/+0Zh+7MYeWuh94VNXDXFKFyDM9D9yNo7G8E+3J7/ekIkrv3hmJyR2wriPwcaec3MFYM/Lqds9R23cXMWbqw6OiopUZoHrHlQv8qk2UtHFEdvf9sD2OIjOe5tq+ZJi00JDOkXX1tVonbJ0SBkJlcuiGcj1pVMX9kicAeDBkp66itPuccWd1GahYPLrTTd4ya1zMlVQ1qieai7NlNPFaTYbED6niwWmHLA/SiXvZy0/QytaarzKN6AyVPK/nd24BNjgKe5EH7arZTLvXok9tU77xYfVIO5+0xqtnb1O+/AhC5DfOzRfDZKe+jvwQ4TdJWFhWjC59ysSHTl2dax0LVnoT+osiPhUilfk42ry680J+UgVo5fnJy7FqFyl2jQ99oZ+prmzil9o+1sAbse7+j5StEyNZl4bht1KrWFtqkqFd1MZawouF3rqa0W0V/OQbYeCGZFiHYBIgdiJd9pPrmHwkTaUO7133FFfClBse6HZC52fo7q12wQqBZpymTyOC6ucJSRPlCboJnyu4zIeYS+15/xBWyuodVYoqWraCDvH4GTrCDldr/aHpyO6tWGchx7mDckta9cU2oDyUE/sESCxRQ015Md3zqYru1OtJ3PDbhyTTrc3cZs2yCxDiIeTiaYglwnlL2wQBumsq3gDWS4VVSti7N9c5qRVqZuklMsFE9PwOGpUneAdCpyxuovbpA9MmQ99msgsY90xtcG0vZtjFy0fAUyBQ08/6OGOI5yj1Pu7oEv5uNAuL2+iGsI7AMtGBsaEBmX0CSnylBrQzhR0nNwJqRvoGGD3WWBfinZQeKXeCX2dq/nITJpl440IdnS1Vp1/GmFkeYEzW7ngXmnNe8vAn6xorNvz1WvrSnHtwYKpRzUkK5gvy0He3LvB31Y8UXQ+Z0mHdR6nuCO7kkIbmYGqDKLwY8u6EBu9nJR/RivEqvjoXsZ+NfKTt+ZKWJkh2SILs5DIlgc/+p+HL9Y0GmMzNxO4y4oEbSNlI0YNHHoukgZTOW6OfVSOU6jjonNz7IMOLcNxwTkNFb3wwyXehJH7ohg7WjRDxlo8BNxCLaMHlW8W129bS8YulsVYNGBgLoU5djeTgnAqFoVdqzeXlzdvS7tkV8p2ME3Gomyt9bIjPTsaMOOSFLb0jjTspLUHoGAopR7w76jRx1qDutLfcQ121Ptj0VA/GnakYbfT4RUK0o7u5miat+aRbrkIeBXrI+sMw84vFE+JwtIySYqcuaDfjAmqVhhCCeZrRq1f0r5hcBE2tfYiISK3ecE17OVWR5Q9mpDYCcmccdgt1h7Bb14WjA7/oEuC6Mf61CXqjRrjKktGRPOGF8xiYSWJiuDxVlkZwSPeaNrG1My4TB6h+yQcipwaGc1IfvV2zyHZfPUQJYeks6l39KdjpMLsmdwSIsW+pEFCOXc6zjug1S2A/+ZmQpVsvZY8gK7Lc2IH1ISzRyC/3V8/XN0Tqcj91dnl1f3JkMBBLJiAqf1gOPxXNFnWrnRVITzv3XwnjrLm1W10bYs1AUzSTQBFOqf+SJlGd9pD7pPmhbWq7qqDBKlCCL/jPe+xe7I7MBKZ5dSwGePMrNbcaq9dK0/qgssZ5dN0Vh4skE7RtJkyuduZuoH061h5/QOnJZdeGTSf93bel1YAqzcAuWKZPWirl8LdtzaupoLTLvXvb8kdq7ZcAGwO6sh8qQRGQSrtKebc1QBHxRxxZkaDIQeRHlscmE0zFOXhlfdWpHO6cE9HSzhiEVzadfKwpUHpqfaDn45Ip08ZOYy+2i3yPtRNM/p1OArjtK46SXFBrCZ4p4utSm9fjwdzoRHR349UJgYmlYnXQOqMJo/4LHmaLKlYwNRXYTpNFLjtqvq87EOzO8upiZu6LACFU4fKXnP2BD63U6M5gbkQm06mXrK0kWpYizUxRb1hUh9ZtWSO7Ql4ZiKVz6dunkH9nM6Kc77DTUWFm99dq1X0Nj/flgreF/s7VJrCM1Bq1sG0VrjOKOehv/86kudYg8I1dHUlxcJEPal6Li/CJw7R5LHIpwqMte+lmPqiZEMe+w8dRS3cvGWORnmDidJnJNFFnkvlmJRLJsw7Jt6hEakANweZAzWFArQW6xekldB+q8NEJYFrBaHGGi1orpfSvBgvfH1Q3I2U80BewOX0DO1wWTCxnqWA3ZN3YkBCkyVMl8xM0RQ9nRV29w1Ie/3ZVbv+kS9X4988uekdqu0Au7piUw1Dbt/dQN8jBA1mHW7vMxY57tMdsoh397pKZVN7jYWp5973wkN47fmrUj01cuotjtz5mPoLn+6ZC71jCHURAdzBdLy/nMT+cEm/kURiJVohUyjDNRsPuiIPGW1TlzE4dc8XX0o/2O3vnqSuZOHiSy6RMT4Rtoxn+JX1OaUc5mYk4hRklKHDHz3YwDBmqJrZTEIsS6i28/NKvf1hmlLGV2F9vmli3eWVcHOwxpNh/KxcjDEfEE8+HPp+OHkEc6rZHy+Vgom+eymvzqh18QmPrRO3s5amcj6Vs/9AYobfW9ETNDdDBza3izgvlxqfMPZInz8TDpU7P0wkcaE//2uWs3AgutfcIy7WLw8Pd9Xx62rTSLSAXOx28sGv3QlRsKAq5eAfna7ynnO4xL4Y1GJoYP7H1UMDtxWuIHtMdNGwAW9ejIj37vPgeNdcwQ4C+fLq5urhamjUy74MikEw/3J1drmVPG+SBanHFIZPk6Y07IVyTTbHoTgrJJOrm6uLB/IJFx3feVtFN7BUOEqmOqFCHPnxTTOfLhyyHou7O9maHYdQr8AU6rWQH8Acg37Oxtxtde/SzuVrKyB0pHi99ZTKZ8ElTV9mZdyyVBhws213ZLvWXO6Nsc6lwPt+Xw2fkplMe96eF/lLkxsQuDXzZhfedjrjzWI/2V1zgqty/sPXZlGmAcXth69fQ2N2nI64WhSuxuk26+Z2HK2q3QJD1/o7IhX5fi1hP45J2I9fv7q4jDoiYSHfbM6wctPKwA63MYdnneWg3gWZw9BPGRFJZJZj7lkpklhFOi7k1sUCI6tOLuWmxDI9mFM0g1LxrucHGvLBuzkqS4DTXLuMmx7W4FrhRq7Y4S/WsWAHfqJDwft1e7f0B8VhZcq0OGaZssltd5myFyzie+c6ykzYH0NUuLdiECpYZKA1XUDUtKa/RN7k48S32bmnZiggytfgibp4TD5OAi6SusYPrPkINcZ1i4ru0/yjp+WuJGXY2oNtXtkd4N54+z1wOyFG5izZAu2tNJhuhQkuvr/FeJBrzcPSMJvbLd0UuEsn12sJy86LFO+ddiXNNXQciy5UABF2/1LYyEDkrmgZN5Yzn4qhS7DWIaPOKjsmzVb1dm9zREFyyVmylej30fDuWjxRztIzYxSbFQaGLgd/AFVxq8BynG8JLaFiBJ85Asg7V+PtK7Xn9kntt+UvyP+bfLp1NeQTqRQkxqUyZtSs7QqwkYu30uuWPw0fXbcLISN27kj/PaSKPYF4kJf8y6jUIlS8fsukNzY6OgbtpXYepCdjfCqwcrX41lqSe9Ix+Tj5KIVZPshLamCSgzCfJ5eDgE6WVC1c3wbH7nrtSdebjSpTVi70yegJ5SBSik9lzdI//nH16aJTuusK4MuBJt+Xo5p8vx5YmdZXIPP8mNLFTlfYAzyvyXMlv7IM66VXrYgcLCKkeOfCzWlpWPk73g6RrIxYv7gpcLoaLvmqZxPFgKpMAj83pjG1y1MpoCiLLMsgZdQA7wmJlLQIaaZPTLO2dTqMq13XCe4AI3POFsuemEaJ7CiomuwzisET5ZXzt6U8WFEaF2mQ152QBX91XGhlbHW2Cm2fpU90wem9rUDc65cNkHW7WPPQa56m4TBaw0PIcrMKxS/GKQvaYM/Z3XXZNZsakjK3wx13CQ0E9CSmgajU7dEv9Fve83Y8dh8N+yxm8uvE68zauLV3X2yQzkn1ofbunuSH+dN1UDpKC6IGc/ptxdC2Z7x2PaXi3RpT2W/jSF02dgU2PLtq3Sj2Q1V2fDnnNHlcSj5Wx4yy9UvlLa5IZjepNa/ILExPlGzVY14D+1be4/ePCDqcFAie0Cbg8hpMH5r4hiPsrei0zAC9i1er2S4o52N0G6raetszvl4Qz568Lq8Mw440SRBAL8ZQ7H8MnPUW5OUylS8wmyB9vmb4mvNP5kxUKillGQjtGlBrLROGRxtenFXC0xbVp1wcJKhPudhbTP91d/v6z+CHQgjgEzPcvUNU/B+IweFP8bWe/YAlli36hHxHmEjx4akml59+u0U/9Pvoj5/v3K/O/3HnfxJ/ejV5ODu/uZ78cnWJv/yOMF2VH6Oc+7RrBLMmQOfIv6SGbjhct6e/YX/ELYesRHiObIFo06m6K6RWZ6cYzv8PAAD//7DuAn0=" + return "eJzsfVtzI7fx77s/BSov3k1pFXvXTp3yw6nSzbFOtFpZ1MZ+Y8CZJokIA8wCGGnpyoc/hQYwg7nxOkPJ//rvS2KRBH7daDS6G43ud+QRVj8R+qy/IcQww+En8pez3yZ/+YaQFHSiWG6YFD+R//sNIYT8mz7rf5NMpgUHkkjOITGanP02IZkUzEjFxIJkYBRLNJkrmeFnF1wW6TM1yfL0G0IUcKAafiIL+g0hcwY81T/h6O+IoBkENPafWeX2i0oWuf9LB6j6IPFAhi706V/LP4fx5Ow/kJjoz+4PU/fpI6yepUq7P55mNM+ZWPjv/uWvf4m+14nN/XugCzsweaK8AJJTpjx/6LMmCrQsVAL6tEWB/nA6K5JHMKf2v1uUtLGuwXBLMyByTiiZfCB+1NaEKctAaCbFK2HcRxSmGFYL8rd/PfUid/rX079+uyPqVBYzDmOA1sQsqSEKTKEEpG69q71Azu6uyZcC1KpNEmfiEdIpTRJZCNOiKN4QpEP+46FYWvtzv+RsoMn+u74khYaUGElYCsKw+cpDJR7qaSeGhuweiMLJsSKUM6q3BxTAzBjnTCw2MnUNin/7Mf5NEikMZcIuNRDQhmXUQEqSJVUL0GQuFVnJQqEa9IgIEw2NGP6VmnEGhm65vFdhzgs3ZSebuazR26LuI/3KsiLrIcBjX7O+F4VSIJLVvmt81Zo38SOSQrCeSSegnlgCtwfIlh8CB0RS7SpmfczohnGWSWXYH5BeSG06gTQFq29J41Fp1tj49SFbSquTvBIaSaQ2fWOGKS2neyfsZuamGVtDhrnOOYj0NbLMAzsaw2rz9bLrVqqMcsvXz5ou4KwL1wszroJICovxGMzrmbOfj5/F7LUKXgntaKLXmLGfaZa1vxZUGGa6NfzLMQ1X/YvHdhSm1WfsZZo2VJlpSs3+Z5MdgdgR8GRS1qSEJ+tf2fPYLpnunBlEetC8VyLdY1YUgWkKcyaYHWcwOXmEpsxtoqZF0cMSiDbomnqDPFegQRhNKDplllJKdA4JmzNIO3FGTuUq7xLMoSBZE8SOZT21NpA6v2ermpNG+l0estlZIzv4Pi2C2la6VbEEvuZcKlAOL5mtKidYtwzzpDSKD7LNq2Ea5nkWe2XPoIDoRNE8eGZlpOI39M6elyxZVgN0xDfselmSUjafg7L/YenQOU3qtmI94BH+rTPqy3GGc5qsxJXDRrL+vAThvNCI/4TmrCM0sBI0k+nsoNUJgxxpbewPL3HKy/NDXS0/9mCqbVIkCWg9L/g9fClAmxtqrM9zSp+a3hrZ8WBsrz/xMkCfQNkjjLu5rJbRJQ6iHBBtHerANutqn2X0DymqP02MApo1WeGBFF6vxWJmWAYkB8Vkero7QzL6dTSGBHfvNTLkk+BMwLVI4esdqASEoQu4U3KhQOtRxSQvp7MMSWSWc7C/cfqCEgHPZMHljHKiIZEipWpFmAVKmCYzsATTNHWhGUoMnXHop/NOySemmRSQ/qaYgQua04SZ1WfBzLh0iiKbgbI05hUG8mxBkMSjQCtPeysBKcHoUw/9W1F5DzR9aSIV0HRwGi+k0EV2bAKDUqsI7SIu8diIfALVvx1POqfRkqxkQRIqiFE0eSRL+UyyIlna2TDEF/PWLJUsFsu8MHY7FBrWbPJ+luki62VZR0hvB4bpIvuTcunI+qEtWZ264c/HtNFl68/Ep3vIOUuopeyYNhhwmutA+QzMM9izVZAiTzHuzAxkhOY5UDQgmECOlTaHRpvD6uzOmaQA61dawpxGPyFUpM7Cbo9MhTRLUOUv/GRe/284vzv4dwyT7X8M/x4UFZomlu4LKeacJWY0ATzzwqfAuvqeS+84PEFk7aYFWMPNVLgot5sXoemS14kU7qKmK65GquGkY4amGeB0ejdWjKSrpKH8tbLhzF239ZmMhnH2B+63oyiqujewyYgsEB2k1v+29HZeDa8ntn5gvRpqO8+0ncmdrLSB7EopqcY8h3d0XZ1iW4AA1Y4eu39UkF8eHu7Ij999R7ShprAHegoHOLgXUqTM7auLJSSPP1PGrag75CMyp7Ln5jglocZAljtu5aDmUmV2Xwd0bunXbNg7ECkTi+gkvEApOAYJeBq5Q88vI1WAiA0IS1D7KOscdVYY9/MlfQIipCErMGRmVVw02IGWAk0flkoaw+HqCcRoi3zfJf1IHHxNAO1D6NdknUMO5CIH8scW8505EFnMnGXMdEezpCC0zFgjb7S1v6musUQ4Frzt5wHq99cpB3UdP6Yg+GPvI/1qd4VeazIfpiqCwbw+PoJcsd7VDFxW1Wxl17L3PHOjM+2khaQSNCoNmud85dTOuxQyNJotl7RlUzeT1mnWik0PdpQba6K9YoZVEuFI7XZlGzFTOY85TX6Wqs08U7E6obmOc5V6zE6aBiPAA95CXJGeQq9R4V3r8Zs7HY+5IJ222OteEQd51CV51Qvx8rrkI/0aeRkov31+1ZgBjMP8qSVbLKGVv+T+tcZqyP4GOd+Fcb0+2stwrimG3UyLf7Jmj+7JtTIHZxbbTrtfksNMH/F+/Op8cli6wtAX4/+SvMhwY56vrDY73OkPQS/N/kDBAZos3f6QufV3mRSxF+uj0Ggi5saavE8ISVs3kSbLcK15y4yS72bUKjgmtKEigRPyvLTrY6KIQiO9J/y5Iwi+yWF2rMGtNypv3Db4UzLHys2nfAjOWIVjMErYsANLvmgM/bYg2i8alq05saN1HA9rYxEPBPtrAQXcgFiY5UB4G1y1h3tT7sog1jNlBiVQWpPCJySgZB1A0kPp8VbpFQPRVj+orv/2KV6HHJQ/Usib6093k7ckBc6eQIGDXq6l/bB2ys2df+1jeFfnE7/5Tslnu8+emVnGeQZugMnkstyjUvDVJrbEN9KjiKjP016z8Jq8EVV2t5Hk/Y9//2fDMHpbXSeul4JheHNeKG3OKbd6bABuVJj+gTFXTu4KlUsNCOnNIn//9oRUAko+5YZlyI1fLi/JG22+f+supC4kD39Lvn9bJ8bRm4Ld+nPLT9xUdCYx0tclpYmC1Bqdb6ykWRAEn8WUMGqfa/M9QsCJFWSUieiibWYZ1npo2C1yeBmDwUG7YOtCQfurQ7fjtJUTZ/xQzlv63DkuA6kXC8CFuo5MVWs3DUnWdcqPQdBajC4PTUi/fqpNsTOSi1nGjInv/ksbPXl/mI2evD+mjX7x/jAbPcmLU+T0ad5KDHfE64RySKdzLmnzC1vkFtc1CeVcJngHf3XxHuWuMBCHBqgC/8bPcOtUkUJDuB8NxmL3eztLiFNCU3z000nLpgePPfnRpQxe3H0uNV25sWJseBDbbxWR47sJ78wdHqMgBopvjGPgjtGiwryk2vqsqoCUaGb/wgx5pppwWgg03FGnU2WayTIxMbpQOS/09AhE+anqFOHlFF5KVSpPkEJg5CjyNZyKsD+7uPt8gSP409u/wmea/AFKbkupnrp3oN3vqQ8mFWnpJNjuFSENySlLSSqfhSW5vd7OGnBqxSwLq0CTAq1FmpbXmI6EnlfaYJ6lejxl4jSn9tDe7zFxN6VNLe9nIAoSYE9W9ASeXB4EYcKAmtMEdGvrMRFKT1hjpssp7KdomoOaakhG0IBt2iIzH3W5tbq2JnM9RbIwR1yk3dHvsUgRSf9TVomJ09nKbP8o35noP5GuH+2xfDjM0XYYznaUlXN0Reu2O4mbRfHlF+5ou+4FV26oHZcy/cjkqfUGjrdyuGphk1Fv5lsqyvXQRiqo4qNPlHG8WTBy33VrETrSup1XZEXLtTeFa4lB3+1Fli1OazrKukWkjrpwgbBo7fakcbMYNmsUrV24rRanClQ0wzPH3mJI29qV2p3Gi17qhthpu8R2OoVzzOVsx6WOu/HGXc4WdYfvvn1W06XmniZLSB6nLr11IFLvIZfKaOtZYwpoDemSapJTjcke0izrH4Z0YYvJP6MAojERuv6Zjx1zqg3JmCjM9kRO3XhHpnUMQsI8L0BK94ptS0x5aCRSrdMk1rxbQPPdze4hOql8jbLNJ1b5KcvoAk6HLItngV1fhps7HL8sS+dCa7vgqyLBp3YNBqwBcS1SlmCSeJCEFIxLf4/Cz0wTEFYX9SjUEmiu2BM1cJoKPR22wh8GlN3o5PJ24uqzefa2PIQtUbJmFoqXxOafd4B2fff0A6FpqkBrQrWWCcOYN97q7YW1mHGWjMVQHLzFzy2l0kMbkIuBcR7HlVUuLCHXd+UnbyyD35KZLNwBug9LcQudJjLt5ubeigjHbfLwxGXCf//3dzNmSCE0WwiMSOMkWyEdft07kZI3uXuwQv5LVCGE+396WRjDxOIdRpn/SwyojAmU6f9aiwULAoX/C+nbDRSZpTVwnaNjVfVYR4GfB82tcCx0XPjxwyrXAD9m0Zqrm+56NS+WlHdOk0cQ6YUUwlndAz1gqy9lUg4fs1VIExVl4SsC2tAZZ3ppjU3/ChMNFElT4m+kVGlnKlgwbTC7JsjmmhzhXx4e7i5kClNP8fT9778PTCW+onv/++9Egc6l0ODe0YXHd5i0eiDoD+OA/jAq6B/GAf3DqKB/HAf0j6OAvro5H5PLCWdWh4FVDQha11G39uiWkEfksQb1BGoQyP6t2TAPP5sJkj4PsoqlINxKW2a07yUumkpPlK95kZwzzuUTqOGgt/Nmwzu8UquXT+9nkNBCu6xgXSgssAnugt6q+zUyApSb5eoXGZh+6LuXOtOXbvhqg8W7Do18rDqypXRMLGVxEu0QYHvZ/AYFnFu0AtTbprS8ebiIPy3zDIJVqGQR0m1piw/9NH4WIy9JIYZdlOHKvVSrgflpvjbJCWEiZLSdOLMQs3vtV9oGCxqApnq779jfoepJIQzjrYCNMi7ooKG0fPwBsgSaglpzQpQl2M9uzs8Sw56gsvTcQg7Doqqqes3o87lgxIplLKcUoTjGucNFB0+wbeuV7K1/ZL9P1QLMluSH9Oebi89DpT13UV0H2Xjz9ebm4vPb+OXcWV4WFiA39pfnG2U7pukWno+3ngKeWwsZW+zHW807Ja3TAIM9JOoj2V9sh+m2X7SyHnb11UMd1fpQR/RZI3JfnfvardPGsHRegTa7wLEfbia3sJCG0dJdH8M0fbiZ1IjECuCx9eydApS4lKXozZfqgFCiQWssLRrCpnWCfREmihOhmb7eaZj+zL5COr33R990DJrndop35elKWxGLKlqxAew9pExBYkaBqfzggwD8rPj0hmXMTK+wcgakR8ScyIKn4ltTf/wVOw6f72/CNVW5LpiEbkXLmT/WoeB27yg7qCD/559bup8ffv99FFqjkIoj2mJ1PihSLRVbYPy1Rxls7/CPB7/H7R8S/49j4u+JAQyK/7vvRsT/3XcjAn8/JvD3IwL/MCbwDyMC/2FM4D8MCfz67unvDQN7DHuqw7RuGwn4WtwCWg93xAidHb4Kv5QZybtFEDvctDFY+uIO2msTmx+QoPXyc+/DlWMs0KYLsM5QaZ2UJVZ7cvUXmNEdhXqioV82hl0tyk78LzhcPVFeuOS6ocEVfLO4LNgTuPJ3LjynrNr0BSs8MVSQpSzWbPERokt7xZR2iZKOHNT16iJ6GSqFZilGPH249wVDzuvQleHodkDHJ6ocGsyphjliIOfWTfpKgzg/c/k8ZAhzTQBnzuWzJm/qlydv2+fjpvOuAXz6cHE3Pnh7wo9GwM3kCATcTEYj4PPlEVbg8+VwK/BnPDeOEIdsct/KzJKKVC/pY3BxfIlnfzkuKixV04AQwrBmiIs0hsvRtcZ6pYrGMtN7xGette4PLB8N26oQd0wLbu7R3I7+PT00Ta/EyTghTCS8wGv1h4u7v13fbb6NrUMfbUE64Meiv65NA67Hn2JnxxT5/e2kaQ11F3dTp7um96BhyOB8O2FDgyFv7icPb+vP7d0DsPLyRG4J++rm/EUw75szZTE7YXpxVjv2OlY7tv+vRzSkR/TIBGh2WGVUP8axfCFXZu+fbtJOX+gF+4f+A8w9JFKlejpUesMujdDCw3fsEA1RONCzy/dXOiEZUF2ojU2/+gU6IvTaWC0j1dkCPjLOmU+tGpf0Rfl8Ah/AKcSCLzM5j8CRhHLuEzHpwsqWIXQ4bth/ZwvMirS/Cl2HE6hlwAfXA4dyBYFAtLB7chrYsRZU9CTeuC7VQLOt1uYYrdNc4hx99E/bIwLKZ7fDCpz/36P0cVGelI49pZdUpcNS5rvtHoWyqKNu15L5l9JD6YtrkciMicX4WrFVsiV+pJIXJuyiuhLYRJirhO+OC+88YB0UOwNKxF3heYg7vPyvmKObuRMk+zj8CbI9Joe8csNnx0Nwqvz6Ec7XllPWy5pChxT/iriqYPK+e6ai9QXUeAchA6iBiqSg6o7XMTtSeK5zmHMrBjeNKol+QRtwJ1nVQwrrcYyOQHef1A5rfETEbSO3hx3RTR+5U0P6ov3UEOv0GN9ODvcryjmkkJ4gS8aUb18iemxzrB018AeXtaqxGkunKMfUp9TQUVgwKbXKMc3SSJcFZrwwH0KbyJcwzX0ihs9Kxrd1gnKMRBYKXpw1UXfBl+eOcWBCi+Rjs+UeaBp3LSqr2IcE5hEVa8WcVoDAhCUqK9dsZ/T20jkpZhbTDB7kxPqJ03tqYHQaIwNcE3Bl1l20geJNj3aocJTQXhW3iY6zmOy5whXQdGWHwaZK+Eai9msfUsbGyb7ngyJSEYZ93+NOjtE1LPI6qsBFOZfPJdNZJII7cHbsE7liathTcdGmJpw6l56p7q2SsD2J2I1zC2NyqO3RKFI7eMSjmz4XPDyHJROpNSH1+mSSw4gdIlTXWnqwdOwTsetmyMscF8dd9ONt3kgjwhOoVVhjv2pMu2pMeNUdb9lTco2fSmG3b6xTUVV+26ch+zmB7Ude/hDcwkLY7TDsjgDFw+0c/gks4zSbpfSgayo3xBEz9m5wwteVrXctnvz7q+ETlqwoaJeLNC9E9XAKd95XSArj3t6HxIvIh3Efu/eYIo3/ExdGgS649/TKoTc8O/SlkIYmklUM3B/bJdD0BowBNRjKn6UiVK9EslRSSOw0EYCeNKwwt05OOms99rFEQakQMTiWAk3fcYTqC4DMCm8xriNPGyZw7kvXQ231s3fFXjOlJeitaCy8nTqMgFXd13wZCmq6dpLOQaRldpDdRIGINdkc3rUZcy80SslQbCfpPao1tx7VxfFAcuHW0/fBzyjWxiv3aagG7w4z7YSl/yLZ/mkDay/KtNKrUmMNzuVSAqrqKlVpmEoQwHXs78X6WSjAN5XpSKh/9m0Qf5uQe1h07EaHsAI/A4u7lukWaPXfSqX41jcXCuCrVN5kTbpNZFx1Ujts7s1OK0SkqHWP3JueZOsGz9tQhKtHnkDhs3/7H5xRv0dcayY538RWkrInllbpZs3Gkj1kV53JdmVAbM0M+/RoG1tmwJUsi329PEVWglOq6ivkQkic9y4h075hXMejG2oW1MAzXR1kvlfD9JjwqNvRWH9GY916MoomjwRb0lkW3J49ED+GNcWpK/fvTotXl0mG0Z5r8bOSWWRPDSwUjUCP37cxn8owQGQf9Ut3BHqCbH0ZvCFJnQkn8P+6u9iA+VNhHuTYfC4b6/jerS3wPlq0PasR9oic9sXP1qLdidnVQ90zZ46P+163MvoxA7CHkm3gXlVx27GfGMdvLnZGjA7lnVTmjIdKK6McJE1hwGIwWEYonOaEBkM8l2qNER1a18piZGHwVplRVGjsvRhHOUMADwtzh4YfKfd/WXOeu6T1SyXzMdCHnPhUYXXvDo23EdrYZ0ira+TBp0gN+CjabWvMOyk3j3vks6TVAHKI0ySGPirHBz9RuovIjVHHNXpE6rVFs17HRm0dQKv0sCcXKj3acwvrzl9ODotiY9f50JV4n2bXobWcT+j7Zs3KrW2F7e7+e1r7O5jvXHP8WuP9shVUd3OFg/p4D0Wa93F27X/9v/26j92vO6WGzqiGaaQ5RiEnTNSoBFn30mvIZmWXq1OqRCeovVqe+Ddb997XJrc0gzdn97dvUQSAJkurETeDSjjV3bzaC9ZFrEBF1H8nNMenIiUZZFKtqvf3iCF88fJ8U0/GCD1LQRg2Z63GKkOQQO2yqne6yHPOIK0Wv5rV389WfwjPlgrBvhRgATh5L79hh92JRNegbDjyJv6eWdeSM6LuOT4/zVLag47pxyneXE1TyM2yE9vBPXRlYTBuZk/Q60+avFFA07+53oPhauQteaasLOCOV5/+0Zh+7MYeWuh94VNXDXFKFyDM9D9yNo7G8E+3J7/ekIkrv3hmJyR2wriPwcaec3MFYM/Lqds9R23cXMWbqw6OiopUZoHrHlQv8qk2UtHFEdvf9sD2OIjOe5tq+ZJi00JDOkXX1tVonbJ0SBkJlcuiGcj1pVMX9kicAeDBkp66itPuccWd1GahYPLrTTd4ya1zMlVQ1qieai7NlNPFaTYbED6niwWmHLA/SiXvZy0/QytaarzKN6AyVPK/nd24BNjgKe5EH7arZTLvXok9tU77xYfVIO5+0xqtnb1O+/AhC5DfOzRfDZKe+jvwQ4TdJWFhWjC59ysSHTl2dax0LVnoT+osiPhUilfk42ry680J+UgVo5fnJy7FqFyl2jQ99oZ+prmzil9o+1sAbse7+j5StEyNZl4bht1KrWFtqkqFd1MZawouF3rqa0W0V/OQbYeCGZFiHYBIgdiJd9pPrmHwkTaUO7133FFfClBse6HZC52fo7q12wQqBZpymTyOC6ucJSRPlCboJnyu4zIeYS+15/xBWyuodVYoqWraCDvH4GTrCDldr/aHpyO6tWGchx7mDckta9cU2oDyUE/sESCxRQ015Md3zqYru1OtJ3PDbhyTTrc3cZs2yCxDiIeTiaYglwnlL2wQBumsq3gDWS4VVSti7N9c5qRVqZuklMsFE9PwOGpUneAdCpyxuovbpA9MmQ99msgsY90xtcG0vZtjFy0fAUyBQ08/6OGOI5yj1Pu7oEv5uNAuL2+iGsI7AMtGBsaEBmX0CSnylBrQzhR0nNwJqRvoGGD3WWBfinZQeKXeCX2dq/nITJpl440IdnS1Vp1/GmFkeYEzW7ngXmnNe8vAn6xorNvz1WvrSnHtwYKpRzUkK5gvy0He3LvB31Y8UXQ+Z0mHdR6nuCO7kkIbmYGqDKLwY8u6EBu9nJR/RivEqvjoXsZ+NfKTt+ZKWJkh2SILs5DIlgc/+p+HL9Y0GmMzNxO4y4oEbSNlI0YNHHoukgZTOW6OfVSOU6jjonNz7IMOLcNxwTkNFb3wwyXehJH7ohg7WjRDxlo8BNxCLaMHlW8W129bS8YulsVYNGBgLoU5djeTgnAqFoVdqzeXlzdvS7tkV8p2ME3Gomyt9bIjPTsaMOOSFLb0jjTspLUHoGAopR7w76jRx1qDutLfcQ121Ptj0VA/GnakYbfT4RUK0o7u5miat+aRbrkIeBXrI+sMw84vFE+JwtIySYqcuaDfjAmqVhhCCeZrRq1f0r5hcBE2tfYiISK3ecE17OVWR5Q9mpDYCcmccdgt1h7Bb14WjA7/oEuC6Mf61CXqjRrjKktGRPOGF8xiYSWJiuDxVlkZwSPeaNrG1My4TB6h+yQcipwaGc1IfvV2zyHZfPUQJYeks6l39KdjpMLsmdwSIsW+pEFCOXc6zjug1S2A/+ZmQpVsvZY8gK7Lc2IH1ISzRyC/3V8/XN0Tqcj91dnl1f3JkMBBLJiAqf1gOPxXNFnWrnRVITzv3XwnjrLm1W10bYs1AUzSTQBFOqf+SJlGd9pD7pPmhbWq7qqDBKlCCL/jPe+xe7I7MBKZ5dSwGePMrNbcaq9dK0/qgssZ5dN0Vh4skE7RtJkyuduZuoH061h5/QOnJZdeGTSf93bel1YAqzcAuWKZPWirl8LdtzaupoLTLvXvb8kdq7ZcAGwO6sh8qQRGQSrtKebc1QBHxRxxZkaDIQeRHlscmE0zFOXhlfdWpHO6cE9HSzhiEVzadfKwpUHpqfaDn45Ip08ZOYy+2i3yPtRNM/p1OArjtK46SXFBrCZ4p4utSm9fjwdzoRHR349UJgYmlYnXQOqMJo/4LHmaLKlYwNRXYTpNFLjtqvq87EOzO8upiZu6LACFU4fKXnP2BD63U6M5gbkQm06mXrK0kWpYizUxRb1hUh9ZtWSO7Ql4ZiKVz6dunkH9nM6Kc77DTUWFm99dq1X0Nj/flgreF/s7VJrCM1Bq1sG0VrjOKOehv/86kudYg8I1dHUlxcJEPal6Li/CJw7R5LHIpwqMte+lmPqiZEMe+w8dRS3cvGWORnmDidJnJNFFnkvlmJRLJsw7Jt6hEakANweZAzWFArQW6xekldB+q8NEJYFrBaHGGi1orpfSvBgvfH1Q3I2U80BewOX0DO1wWTCxnqWA3ZN3YkBCkyVMl8xM0RQ9nRV29w1Ie/3ZVbv+kS9X4988uekdqu0Au7piUw1Dbt/dQN8jBA1mHW7vMxY57tMdsoh397pKZVN7jYWp5973wkN47fmrUj01cuotjtz5mPoLn+6ZC71jCHURAdzBdLy/nMT+cEm/kURiJVohUyjDNRsPuiIPGW1TlzE4dc8XX0o/2O3vnqSuZOHiSy6RMT4Rtoxn+JX1OaUc5mYk4hRklKHDHz3YwDBmqJrZTEIsS6i28/NKvf1hmlLGV2F9vmli3eWVcHOwxpNh/KxcjDEfEE8+HPp+OHkEc6rZHy+Vgom+eymvzqh18QmPrRO3s5amcj6Vs/9AYobfW9ETNDdDBza3izgvlxqfMPZInz8TDpU7P0wkcaE//2uWs3AgutfcIy7WLw8Pd9Xx62rTSLSAXOx28sGv3QlRsKAq5eAfna7ynnO4xL4Y1GJoYP7H1UMDtxWuIHtMdNGwAW9ejIj37vPgeNdcwQ4C+fLq5urhamjUy74MikEw/3J1drmVPG+SBanHFIZPk6Y07IVyTTbHoTgrJJOrm6uLB/IJFx3feVtFN7BUOEqmOqFCHPnxTTOfLhyyHou7O9maHYdQr8AU6rWQH8Acg37Oxtxtde/SzuVrKyB0pHi99ZTKZ8ElTV9mZdyyVBhws213ZLvWXO6Nsc6lwPt+Xw2fkplMe96eF/lLkxsQuDXzZhfedjrjzWI/2V1zgqty/sPXZlGmAcXth69fQ2N2nI64WhSuxuk26+Z2HK2q3QJD1/o7IhX5fi1hP45J2I9fv7q4jDoiYSHfbM6wctPKwA63MYdnneWg3gWZw9BPGRFJZJZj7lkpklhFOi7k1sUCI6tOLuWmxDI9mFM0g1LxrucHGvLBuzkqS4DTXLuMmx7W4FrhRq7Y4S/WsWAHfqJDwft1e7f0B8VhZcq0OGaZssltd5myFyzie+c6ykzYH0NUuLdiECpYZKA1XUDUtKa/RN7k48S32bmnZiggytfgibp4TD5OAi6SusYPrPkINcZ1i4ru0/yjp+WuJGXY2oNtXtkd4N54+z1wOyFG5izZAu2tNJhuhQkuvr/FeJBrzcPSMJvbLd0UuEsn12sJy86LFO+ddiXNNXQciy5UABF2/1LYyEDkrmgZN5Yzn4qhS7DWIaPOKjsmzVb1dm9zREFyyVmylej30fDuWjxRztIzYxSbFQaGLgd/AFVxq8BynG8JLaFiBJ85Asg7V+PtK7Xn9kntt+UvyP+bfLp1NeQTqRQkxqUyZtSs7QqwkYu30uuWPw0fXbcLISN27kj/PaSKPYF4kJf8y6jUIlS8fsukNzY6OgbtpXYepCdjfCqwcrX41lqSe9Ix+Tj5KIVZPshLamCSgzCfJ5eDgE6WVC1c3wbH7nrtSdebjSpTVi70yegJ5SBSik9lzdI//nH16aJTuusK4MuBJt+Xo5p8vx5YmdZXIPP8mNLFTlfYAzyvyXMlv7IM66VXrYgcLCKkeOfCzWlpWPk73g6RrIxYv7gpcLoaLvmqZxPFgKpMAj83pjG1y1MpoCiLLMsgZdQA7wmJlLQIaaZPTLO2dTqMq13XCe4AI3POFsuemEaJ7CiomuwzisET5ZXzt6U8WFEaF2mQ152QBX91XGhlbHW2Cm2fpU90wem9rUDc65cNkHW7WPPQa56m4TBaw0PIcrMKxS/GKQvaYM/Z3XXZNZsakjK3wx13CQ0E9CSmgajU7dEv9Fve83Y8dh8N+yxm8uvE68zauLV3X2yQzkn1ofbunuSH+dN1UDpKC6IGc/ptxdC2Z7x2PaXi3RpT2W/jSF02dgU2PLtq3Sj2Q1V2fDnnNHlcSj5Wx4yy9UvlLa5IZjepNa/ILExPlGzVY14D+1be4/ePCDqcFAie0E2AcauMjNff8g2OdiyhWIN3C5EoLxr1oamFOMLeR4mWGaD/9mrPjgvK+Rj9nKrG6daKqpcctLaNy9zDwC5NEgTQizG0UxgDZ73Je7lM5RvXJkifERu+5jzAOROV0k9ZBkK7Ft9ay4Sh8YBXk5XwtEX1KRcHCepTLvYW03/d3b5+K+ehEAL4xAx3sxO1VwBicPhTfA9pP2CJZYs+Id8RJlJ82qvJ5affbtHT/z764+c796vzf9z5n8SfXk0ezs5vrie/XF3iL78jTFcF3ijnPrEdwawJgTryL6mhG8yX7elvWHhxUycrEZ4jWyDaZLfsCqnVOyuG8/8DAAD//0xoafA=" } diff --git a/x-pack/metricbeat/module/aws/transitgateway/_meta/fields.yml b/x-pack/metricbeat/module/aws/transitgateway/_meta/fields.yml index e687ae973d0..a47b2f4bf54 100644 --- a/x-pack/metricbeat/module/aws/transitgateway/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/transitgateway/_meta/fields.yml @@ -25,3 +25,9 @@ - name: PacketDropCountNoRoute.sum type: long description: The number of packets dropped because they did not match a route. + - name: BytesDropCountNoRoute.sum + type: long + description: The number of bytes dropped because they did not match a route. + - name: BytesDropCountBlackhole.sum + type: long + description: The number of bytes dropped because they matched a blackhole route. \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/transitgateway/manifest.yml b/x-pack/metricbeat/module/aws/transitgateway/manifest.yml index 66cf33e3071..8a8f215fe66 100644 --- a/x-pack/metricbeat/module/aws/transitgateway/manifest.yml +++ b/x-pack/metricbeat/module/aws/transitgateway/manifest.yml @@ -13,3 +13,5 @@ input: - PacketsOut - PacketDropCountBlackhole - PacketDropCountNoRoute + - BytesDropCountNoRoute + - BytesDropCountBlackhole diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 50cdc1f2ae1..bef034013bb 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -18,22 +18,22 @@ import ( resourcegroupstaggingapitypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" ) -// GetStartTimeEndTime function uses durationString to create startTime and endTime for queries. -func GetStartTimeEndTime(period time.Duration, latency time.Duration) (time.Time, time.Time) { - endTime := time.Now() - if latency != 0 { - // add latency if config is not 0 - endTime = endTime.Add(latency * -1) - } - - // Set startTime to be one period earlier than the endTime. If metrics are - // not being collected, use latency config parameter to offset the startTime - // and endTime. - startTime := endTime.Add(period * -1) - // Defining duration - d := 60 * time.Second - // Calling Round() method - return startTime.Round(d), endTime.Round(d) +// GetStartTimeEndTime calculates start and end times for queries based on the current time and a duration. +// +// Whilst the inputs to this function are continuous, the maximum period granularity we can consistently use +// is 1 minute. The resulting interval should also be aligned to the period for best performance. This means +// if a period of 3 minutes is requested at 12:05, for example, the calculated times are 12:00->12:03. See +// https://github.com/aws/aws-sdk-go-v2/blob/fdbd882cdf5c63a578caed14688cf9a456c75f2b/service/cloudwatch/api_op_GetMetricData.go#L88 +// for more information about granularity and period alignment. +// +// If durations are configured in non-whole minute periods, they are rounded up to the next minute e.g. 90s becomes 120s. +// +// If `latency` is configured, the period is shifted back in time by specified duration (before period alignment). +func GetStartTimeEndTime(now time.Time, period time.Duration, latency time.Duration) (time.Time, time.Time) { + periodInMinutes := (period + time.Second*29).Round(time.Second * 60) + endTime := now.Add(latency * -1).Truncate(periodInMinutes) + startTime := endTime.Add(periodInMinutes * -1) + return startTime, endTime } // GetListMetricsOutput function gets listMetrics results from cloudwatch ~~per namespace~~ for each region. diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index 1d6862f6423..3329ee45e2d 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -160,7 +160,7 @@ func TestGetListMetricsOutputWithWildcard(t *testing.T) { } func TestGetMetricDataPerRegion(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) + startTime, endTime := GetStartTimeEndTime(time.Now(), 10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} var metricDataQueries []cloudwatchtypes.MetricDataQuery @@ -194,7 +194,7 @@ func TestGetMetricDataPerRegion(t *testing.T) { } func TestGetMetricDataResults(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) + startTime, endTime := GetStartTimeEndTime(time.Now(), 10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} metricInfo := cloudwatchtypes.Metric{ @@ -434,3 +434,135 @@ func TestGetResourcesTags(t *testing.T) { } assert.Equal(t, expectedResourceTagMap, resourceTagMap) } + +func parseTime(t *testing.T, in string) time.Time { + time, err := time.Parse(time.RFC3339, in) + if err != nil { + t.Errorf("test setup failed - could not parse time with time.RFC3339: %s", in) + } + return time +} + +func TestGetStartTimeEndTime(t *testing.T) { + var cases = []struct { + title string + start string + period time.Duration + latency time.Duration + expectedStart string + expectedEnd string + }{ + // window should align with period e.g. requesting a 5 minute period at 10:27 gives 10:20->10:25 + {"1 minute", "2022-08-15T13:38:45Z", time.Second * 60, 0, "2022-08-15T13:37:00Z", "2022-08-15T13:38:00Z"}, + {"2 minutes", "2022-08-15T13:38:45Z", time.Second * 60 * 2, 0, "2022-08-15T13:36:00Z", "2022-08-15T13:38:00Z"}, + {"3 minutes", "2022-08-15T13:38:45Z", time.Second * 60 * 3, 0, "2022-08-15T13:33:00Z", "2022-08-15T13:36:00Z"}, + {"5 minutes", "2022-08-15T13:38:45Z", time.Second * 60 * 5, 0, "2022-08-15T13:30:00Z", "2022-08-15T13:35:00Z"}, + {"30 minutes", "2022-08-15T13:38:45Z", time.Second * 60 * 30, 0, "2022-08-15T13:00:00Z", "2022-08-15T13:30:00Z"}, + + // latency should shift the time *before* period alignment + // e.g. requesting a 5 minute period at 10:27 with 1 minutes latency still gives 10:20->10:25, + // but with 3 minutes latency gives 10:15->10:20 + {"1 minute, 10 minutes latency", "2022-08-15T13:38:45Z", time.Second * 60, time.Second * 60 * 10, "2022-08-15T13:27:00Z", "2022-08-15T13:28:00Z"}, + {"2 minutes, 1 minute latency", "2022-08-15T13:38:45Z", time.Second * 60 * 2, time.Second * 60, "2022-08-15T13:34:00Z", "2022-08-15T13:36:00Z"}, + {"5 minutes, 4 minutes latency", "2022-08-15T13:38:45Z", time.Second * 60 * 5, time.Second * 60 * 4, "2022-08-15T13:25:00Z", "2022-08-15T13:30:00Z"}, + {"30 minutes, 30 minutes latency", "2022-08-15T13:38:45Z", time.Second * 60 * 30, time.Second * 60 * 30, "2022-08-15T12:30:00Z", "2022-08-15T13:00:00Z"}, + + // non-whole-minute periods should be rounded up to the nearest minute; latency is applied as-is before period adjustment + {"20 seconds, 45 second latency", "2022-08-15T13:38:45Z", time.Second * 20, time.Second * 45, "2022-08-15T13:37:00Z", "2022-08-15T13:38:00Z"}, + {"1.5 minutes, 60 second latency", "2022-08-15T13:38:45Z", time.Second * 90, time.Second * 60, "2022-08-15T13:34:00Z", "2022-08-15T13:36:00Z"}, + {"just less than 5 minutes, 3 minute latency", "2022-08-15T13:38:45Z", time.Second * 59 * 5, time.Second * 90, "2022-08-15T13:30:00Z", "2022-08-15T13:35:00Z"}, + } + + for _, tt := range cases { + t.Run(tt.title, func(t *testing.T) { + startTime, expectedStartTime, expectedEndTime := parseTime(t, tt.start), parseTime(t, tt.expectedStart), parseTime(t, tt.expectedEnd) + + start, end := GetStartTimeEndTime(startTime, tt.period, tt.latency) + + if expectedStartTime != start || expectedEndTime != end { + t.Errorf("got (%s, %s), want (%s, %s)", start, end, tt.expectedStart, tt.expectedEnd) + } + }) + } +} + +func TestGetStartTimeEndTime_AlwaysCreatesContinuousIntervals(t *testing.T) { + type interval struct { + start, end string + } + + startTime := parseTime(t, "2022-08-24T11:01:00Z") + numCalls := 5 + + var cases = []struct { + title string + period time.Duration + latency time.Duration + expectedIntervals []interval + }{ + // with no latency + {"1 minute", time.Second * 60, 0, []interval{ + {"2022-08-24T11:00:00Z", "2022-08-24T11:01:00Z"}, + {"2022-08-24T11:01:00Z", "2022-08-24T11:02:00Z"}, + {"2022-08-24T11:02:00Z", "2022-08-24T11:03:00Z"}, + {"2022-08-24T11:03:00Z", "2022-08-24T11:04:00Z"}, + {"2022-08-24T11:04:00Z", "2022-08-24T11:05:00Z"}, + }}, + {"2 minutes", time.Second * 60 * 2, 0, []interval{ + {"2022-08-24T10:58:00Z", "2022-08-24T11:00:00Z"}, + {"2022-08-24T11:00:00Z", "2022-08-24T11:02:00Z"}, + {"2022-08-24T11:02:00Z", "2022-08-24T11:04:00Z"}, + {"2022-08-24T11:04:00Z", "2022-08-24T11:06:00Z"}, + {"2022-08-24T11:06:00Z", "2022-08-24T11:08:00Z"}, + }}, + {"3 minutes", time.Second * 60 * 3, 0, []interval{ + {"2022-08-24T10:57:00Z", "2022-08-24T11:00:00Z"}, + {"2022-08-24T11:00:00Z", "2022-08-24T11:03:00Z"}, + {"2022-08-24T11:03:00Z", "2022-08-24T11:06:00Z"}, + {"2022-08-24T11:06:00Z", "2022-08-24T11:09:00Z"}, + {"2022-08-24T11:09:00Z", "2022-08-24T11:12:00Z"}, + }}, + {"5 minutes", time.Second * 60 * 5, 0, []interval{ + {"2022-08-24T10:55:00Z", "2022-08-24T11:00:00Z"}, + {"2022-08-24T11:00:00Z", "2022-08-24T11:05:00Z"}, + {"2022-08-24T11:05:00Z", "2022-08-24T11:10:00Z"}, + {"2022-08-24T11:10:00Z", "2022-08-24T11:15:00Z"}, + {"2022-08-24T11:15:00Z", "2022-08-24T11:20:00Z"}, + }}, + {"30 minutes", time.Second * 60 * 30, 0, []interval{ + {"2022-08-24T10:30:00Z", "2022-08-24T11:00:00Z"}, + {"2022-08-24T11:00:00Z", "2022-08-24T11:30:00Z"}, + {"2022-08-24T11:30:00Z", "2022-08-24T12:00:00Z"}, + {"2022-08-24T12:00:00Z", "2022-08-24T12:30:00Z"}, + {"2022-08-24T12:30:00Z", "2022-08-24T13:00:00Z"}, + }}, + + // with 90s latency (sanity check) + {"1 minute with 2 minute latency", time.Second * 60, time.Second * 90, []interval{ + {"2022-08-24T10:58:00Z", "2022-08-24T10:59:00Z"}, + {"2022-08-24T10:59:00Z", "2022-08-24T11:00:00Z"}, + {"2022-08-24T11:00:00Z", "2022-08-24T11:01:00Z"}, + {"2022-08-24T11:01:00Z", "2022-08-24T11:02:00Z"}, + {"2022-08-24T11:02:00Z", "2022-08-24T11:03:00Z"}, + }}, + } + + for _, tt := range cases { + t.Run(tt.title, func(t *testing.T) { + // get a few repeated intervals + intervals := make([]interval, numCalls) + for i := range intervals { + adjustedStartTime := startTime.Add(tt.period * time.Duration(i)) + start, end := GetStartTimeEndTime(adjustedStartTime, tt.period, tt.latency) + intervals[i] = interval{start.Format(time.RFC3339), end.Format(time.RFC3339)} + } + + for i, val := range intervals { + if val != tt.expectedIntervals[i] { + t.Errorf("got %v, want %v", intervals, tt.expectedIntervals) + break + } + } + }) + } +} diff --git a/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml b/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml index 27f5d1f99a1..2d279172f84 100644 --- a/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/dataproc/_meta/fields.yml @@ -57,3 +57,19 @@ - name: cluster.yarn.virtual_cores.count type: long description: Indicates the number of virtual cores in YARN. + - name: cluster.job.completion_time.value + type: object + object_type: histogram + description: The time jobs took to complete from the time the user submits a job to the time Dataproc reports it is completed. + - name: cluster.job.duration.value + type: object + object_type: histogram + description: The time jobs have spent in a given state. + - name: cluster.operation.completion_time.value + type: object + object_type: histogram + description: The time operations took to complete from the time the user submits a operation to the time Dataproc reports it is completed. + - name: cluster.operation.duration.value + type: object + object_type: histogram + description: The time operations have spent in a given state. diff --git a/x-pack/metricbeat/module/gcp/dataproc/manifest.yml b/x-pack/metricbeat/module/gcp/dataproc/manifest.yml index 41d53b47cfa..ed85b4729ea 100644 --- a/x-pack/metricbeat/module/gcp/dataproc/manifest.yml +++ b/x-pack/metricbeat/module/gcp/dataproc/manifest.yml @@ -24,3 +24,7 @@ input: - "cluster/yarn/nodemanagers" - "cluster/yarn/pending_memory_size" - "cluster/yarn/virtual_cores" + - "cluster/job/completion_time" + - "cluster/job/duration" + - "cluster/operation/completion_time" + - "cluster/operation/duration" diff --git a/x-pack/metricbeat/module/gcp/distribution.go b/x-pack/metricbeat/module/gcp/distribution.go new file mode 100644 index 00000000000..aae21e1d58e --- /dev/null +++ b/x-pack/metricbeat/module/gcp/distribution.go @@ -0,0 +1,97 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package gcp + +import ( + "math" + + "google.golang.org/genproto/googleapis/api/distribution" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func containsHistogram(d *distribution.Distribution) bool { + if d.Count == 0 || d.BucketOptions == nil || len(d.BucketCounts) == 0 { + return false + } + + var sum int64 + + for _, v := range d.BucketCounts { + sum += v + } + + // Count must be equal to the sum of values in BucketCounts. + return d.Count == sum +} + +// Explicit: You list all the boundaries for the buckets in the bounds array. +// Bucket i has these boundaries: +// Upper bound: bounds[i] for (0 <= i < N-1) +// Lower bound: bounds[i - 1] for (1 <= i < N) +// https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TypedValue#Explicit + +func calcExplicitUpperBound(bucket *distribution.Distribution_BucketOptions_Explicit, i int) float64 { + return bucket.Bounds[i] +} + +// Exponential(scale, growth_factor, i): Bucket widths increase for higher values. +// The boundaries are scale * growth_factor**i, for i=0,1,2,...,N. +// https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TypedValue#Exponential + +func calcExponentialUpperBound(bucket *distribution.Distribution_BucketOptions_Exponential, i int) float64 { + return bucket.Scale * (math.Pow(bucket.GrowthFactor, float64(i))) +} + +// Linear(offset, width, i): Every bucket has the same width. +// The boundaries are offset + width * i, for i=0,1,2,...,N. +// https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TypedValue#Linear + +func calcLinearUpperBound(bucket *distribution.Distribution_BucketOptions_Linear, i int) float64 { + return bucket.Offset + (bucket.Width * float64(i)) +} + +func createHistogram(values []float64, counts []uint64) mapstr.M { + return mapstr.M{ + "values": values, + "counts": counts, + } +} + +func DistributionHistogramToES(d *distribution.Distribution) mapstr.M { + if !containsHistogram(d) { + return createHistogram([]float64{}, []uint64{}) + } + + values := make([]float64, 0, len(d.BucketCounts)) + counts := make([]uint64, 0, len(d.BucketCounts)) + + switch { + case d.BucketOptions.GetExplicitBuckets() != nil: + bucket := d.BucketOptions.GetExplicitBuckets() + + for i := range d.BucketCounts { + values = append(values, calcExplicitUpperBound(bucket, i)) + } + case d.BucketOptions.GetExponentialBuckets() != nil: + bucket := d.BucketOptions.GetExponentialBuckets() + + for i := range d.BucketCounts { + values = append(values, calcExponentialUpperBound(bucket, i+1)) + } + case d.BucketOptions.GetLinearBuckets() != nil: + bucket := d.BucketOptions.GetLinearBuckets() + + for i := range d.BucketCounts { + values = append(values, calcLinearUpperBound(bucket, i+1)) + } + } + + for i := range d.BucketCounts { + counts = append(counts, uint64(d.BucketCounts[i])) + } + + return createHistogram(values, counts) +} diff --git a/x-pack/metricbeat/module/gcp/distribution_test.go b/x-pack/metricbeat/module/gcp/distribution_test.go new file mode 100644 index 00000000000..e2f582653be --- /dev/null +++ b/x-pack/metricbeat/module/gcp/distribution_test.go @@ -0,0 +1,129 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package gcp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/genproto/googleapis/api/distribution" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestDistributionHistogramToES(t *testing.T) { + type sample struct { + histogram *distribution.Distribution + expected mapstr.M + } + + // Histogram samples copied from: + // https://cloud.google.com/logging/docs/logs-based-metrics/distribution-metrics + + cases := map[string]struct { + samples []sample + }{ + "explicit histogram": { + samples: []sample{ + { + histogram: &distribution.Distribution{ + BucketCounts: []int64{0, 0, 0, 6, 1, 1}, + Count: 8, + BucketOptions: &distribution.Distribution_BucketOptions{ + Options: &distribution.Distribution_BucketOptions_ExplicitBuckets{ + ExplicitBuckets: &distribution.Distribution_BucketOptions_Explicit{ + Bounds: []float64{0, 1, 2, 5, 10, 20}, + }, + }, + }, + }, + expected: mapstr.M{ + "counts": []uint64{0, 0, 0, 6, 1, 1}, + "values": []float64{0, 1, 2, 5, 10, 20}, + }, + }, + }, + }, + "exponential histogram": { + samples: []sample{ + { + histogram: &distribution.Distribution{ + BucketCounts: []int64{0, 0, 3, 1}, + Count: 4, + BucketOptions: &distribution.Distribution_BucketOptions{ + Options: &distribution.Distribution_BucketOptions_ExponentialBuckets{ + ExponentialBuckets: &distribution.Distribution_BucketOptions_Exponential{ + NumFiniteBuckets: 4, + Scale: 3, + GrowthFactor: 2, + }, + }, + }, + }, + expected: mapstr.M{ + "counts": []uint64{0, 0, 3, 1}, + "values": []float64{6, 12, 24, 48}, + }, + }, + }, + }, + "linear histogram": { + samples: []sample{ + { + histogram: &distribution.Distribution{ + BucketCounts: []int64{0, 1, 2, 0}, + Count: 3, + BucketOptions: &distribution.Distribution_BucketOptions{ + Options: &distribution.Distribution_BucketOptions_LinearBuckets{ + LinearBuckets: &distribution.Distribution_BucketOptions_Linear{ + NumFiniteBuckets: 4, + Offset: 5, + Width: 15, + }, + }, + }, + }, + expected: mapstr.M{ + "counts": []uint64{0, 1, 2, 0}, + "values": []float64{20, 35, 50, 65}, + }, + }, + }, + }, + "no histogram": { + samples: []sample{ + { + histogram: &distribution.Distribution{ + BucketOptions: &distribution.Distribution_BucketOptions{ + Options: &distribution.Distribution_BucketOptions_LinearBuckets{ + LinearBuckets: &distribution.Distribution_BucketOptions_Linear{ + NumFiniteBuckets: 4, + Offset: 5, + Width: 15, + }, + }, + }, + }, + expected: mapstr.M{ + "counts": []uint64{}, + "values": []float64{}, + }, + }, + }, + }, + } + + for title, c := range cases { + t.Run(title, func(t *testing.T) { + for i, s := range c.samples { + histogram := s.histogram + t.Logf("#%d: %+v", i, histogram) + + result := DistributionHistogramToES(histogram) + assert.EqualValues(t, s.expected, result) + } + }) + } +} diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index 6e33b82b000..55948c649a6 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzcXVuPG7eSfvevIPJiezHp7Mku9sFYHMAen+QY68kZ7NgB9qnDJksSLTbZ4UVj5dcveOuL1Bq1+qKJYz9p1M366kZWFYvU92gL+zdoTaoXCBlmOLxBL3+Wcs0B3XJpKbrn2KykKl++QEgBB6zhDSrA4BcIUdBEscowKd6gv79ACKGfb+9RKanl8AKhFQNO9Rv/xfdI4BISKffP7Cv3WUmb/tJ+vv0OxwVwXf85vSqLL0BM6889eNK/gEswIxUTa1SCUYzo45EPIbRhWA0q+7fOVyehuH/hj3l4Ygv7R6lo78AlGEyxwUsN7lhdZGy91wbKRYZWoKVVBGYbPA38XS2Q8P+783bVGZdKW3jr7vk2L3FVMbGOj37XGfwJ67yL5mg22CAFxioBFK2ULFHHGd/ef0C/W1D77IitgnHOxPoUvc4w78KzyTRa7xx6eFcwbV9Fvc6S0BCpg0ReHKuuT+sdrLdSG/+sRkwQbikgBWvLsbpBBn+9QZh+sdqUIMwNwoIiJa2gTuyglFRZDx4mdpIRyEspzGYMpiQyBZVUBvlx+ghVSnprYHQMlfvwNvrwHskVMhtIak10C+BSrDUyso+4kQbzHrorLrE5TfWTe62mhEtphTk2MCLLyhoYZGC34dkFDWzFFDxizjOqZFUBzYq9Ad3Du5PXadY/CCJLx7V/HcXBULH3wk9EBtDPK0y2YHROvPR2mNs+2x+IJg52AR4mtMGCQEYqmynQoHZAcyIV6JNgjmayAzi/2LIA5SzRj4PSsEgKD2fj/DTaaaJ/DprVeA25YSVkGsgIUJ/dAGglFcKcR2BMIA1ECqoHkc+qg5VjGOVPTgMKE/cpsY05lwQboOj2/nOYvplGxCoFwvC9Q2Y1JIENERJlepspwGMt+tbZn4MXLNqNFBYSN/AgwrmspplxDcENGRB8+BeSFSjsHnhSSR7Fo2IG5uHfDWVAICOHCcCTnlkCfszhIiihlGqfFc62pMgULnPN/oCRUJzV+nUhTuwOVaDgjNNZ5a93Gfq0YTrO1s6ApeB7hHeYcVzw4G2/3sXwJCxBTqDuZfgRrXDJ+L5/1T3NktVAR7J0F+A3XubGej5u9COuciZG2qvTz5FmvM8wEVGtLWgTnJgZjeSjQI4m0hUm8CzcSmvmZDc5qWex4djI5+BXgHmUapsxsVag9VzTMAG2SzG9AxPJXIIkRgWZn5nGI0rBxSRMMKNwNLhPO1AjQcwsl4vh2OqJeOZp4k2IFQOYOIehDdaoABBIWSGYWD9psgFA7qf5UTD+wXHl5lA3DNJMEEg4HrFG2mBlgN604qwMPeCy4kAR7EDt0X/9e/PN25UBhbT7non1DaLYYOeoQhq0Y5olL7WVc8y//di8epRzuFcrJcmgpON9fHiBrKOVZmNDNpmusDO/r0CskarP8k7K/Sjkp8wFkEHxojaHd44OenB0UEOnFxLhVhtQ2YaudOYkJiSFJTD98/1PD17KvzgCYYbFCpKJOntkFBCuEZ2Hq41ULiEguMKEmX1PRPBEeH4SdxquRh0KVTVSKRIMZ9M/v7sAqTWMsz98FDcJrFsIK1AEhHEJTQ00kDkIbgbgs2IDmJvNPi+4JNsl9F+TQIFEUrh7apDKv8giW2HGgS6A7ossok1u8A5QoOM0PdAeHbg02S6Jru0xl6HTtihdoHQF6fmlp6bn5umBOOsUZzlFN1nUFHU3QJdT+iHScapvkC5pAL1SHW0Ge6xEVhdH8hDn5818N/PU+X9v//eXOqvVTVVmCMiqWmKuxMSwHQRguKq4e8gJdwAiIoXBTMAicYUH1KJwHk7U3YkSxJjV2SFqa8wNPWQd9nBccFNigdcLysfFN3eRxmF0M9z8KwibEnMJ0Jl9DAuaOsHvLlO+CdK7qQuQkbRz2ALQyvIV47ypI2uyAWr5IC52TBmLeSwlzy/wOH5TyXUaOM4EVkyBi42G7T/8lJ5eNBegktgShMkocDAwQTifOiLRlhDQemV5TQIFEif8tQbi68ZLwnAEdCgY/G5BMdBIKsSl3NrqHLhQ1V0SnafQk0autyfN5uVv6y38lmbEYJxlezf20bmLwlW9H3t7jx4MJluq2A6U35KNb/st1eMeA5frurd+/p9/vJzRDpt9uTiZ+w0O50b55E2WW1tajv0Kdnv/GfnxfMRSb7n4kmucTmoEwwoEZ3ngrGRm4t6Vgx2Q+tHSfk1DaDK+diI434aSwx0QD91MigVRLyhEsBDSIPhKACj6G8I6Kq/7hXvfU5mvjvPjf14gwbhuzbY/2Wg7jtxjnAuXrM6xuoy5NOxesv94aDIuSFgrwMbXPLE4MJy21USCz2A3UG2gBIV5HmskwQ9HlqA/SoI5qses6y7B95gIpemxs8Qx1iS3edHGURfA66tO84IN9jQVatyOmqL7uH/YvzDMCnAhry/bDHxj68TgyTPKsXKhzApbbkbu7zSrROU7R9xQ+gYVSm5BICofhV8q9hXcoBJ/kcq3lJVM9HeSHQGc5tl3nURulDUurIBrLWDRqP/Sa1gU6YTJNdrLPHOpS5Kxmu5YLtPQB4a7wb5hLGwdPoP1Prkpe9ZSWQmtZCYY5TF/nQ3ai3QgJA0NabFUiQsOs0XDrTFbkXH0H095OtZlZoND5MMngyu5ci2Lq+bZLdWNTrFr5L5LYKKthY5h0ZN/TbOy4zi0ZREzh6Oh96RtcXNjZ34jPl8pGNu895OCds0rDOhw8n6m5oQduklGNh0e2Md1gAfEI/vSPOJTVjK3ZSyQXQWg7bLcSLQxRpnueK3JLcZ2C2K8woI0MUAdkm/1kLtu9N+W7hR3aoKR6EB1r+d8E+70YHpBC00tiqnJMg96HNmi2OAUB4KtmzhbcI+aFyfg1yDMUtiNwkLH1oKZ4VeM5qFwMb5rvsRf0X04ifWvh4m26vBM6Hjvbn2lHelKSQJap873qSBziqGU4tm2kbzuOeyAp3a9AGhSyJuYmjnzThvphzhHpOOVpEezxfT57Nw8UUk60c/auN0sMT/mnvlhHtg7yW05R7zYIPYHfWJKUZ9FiOcOHMlrLeAt9ibY+i99fLV99WmWzuBaJFILg/dEZwW4ubIN/qkI7ekq4kFpPJ6rqvUdMcRzKlfpkOcS0wJzLMjQw98fJaboXXplwSO6G2MqnRWYbEHQfFqZvLsAto6N4HrLN7al/PPTp/sfHrxcUBCMU6VEEUevcfYjHRfvHCzVCVs8OVvsayC+5bsH7BCAupJCj83LnpZlGDoKs8b6SipEMNnAaydL+GpACcw9/lcPr4cycC0bIJyBMNpBvUzCC6v+UjDXUvMpn4ly7IPI/yNLVjDtLNhJmB5ZbWmfbu9/+Pz+Pq35B1ijnTaY06qw4vIxQz9J5QbwnzRi5qVGXsPNwX8XlDZdukgbBbj0RwuHMZ9PO4PWFULnINp8YjjDyLTzjk+rMaJpTx2DOFlWdZHjhXU3mvd+9Ews73UfPr7rsaVXqwm6eD2MnYV10c/YeSepUV7HSdowl5X6tV2gxVnvBTakyrXmeaXk131GuNT+EhMhgITTE+PKJ+2LTOqxUnuvAmRAlUz4azx8cun88+HhIwowzuKc5ImHldJGar/etWzU6ljxGQJompWeRtTo8de7yxAJeLyCHonP3MYpUVYgZoB4Gw9otA5WWePSTX8eowtbSbve+KnnBNbvT+Zi/ta+49StP0Ubc0/Tqav4KltoWwwa/N4WD7ZYMMHUAld6I41PjLhcTyrluHfDSaS6yU67tN8Fy77lw98wghuiAwCFwnle7HMFayZHXxEyCuBRi1vQSkTyFHoixYqtc1tRPEvdn6QbFsLANpwzRGSDxRr0TdB4yJDqU4ieRrjcDbTlT4tb2DJP0pg8tUzRextIS+0LQxqvackpaJMg53g97haLt2tAr5oq/etkoWH4BH+MQI8BtsR6daiXC9oWNe0Mk21iZEafqm0Dk62QjxzoOrjS2+ZzXYvo+BoFznxp0pE+i36ROdaKDuqal1c422Y4Q5Fo/cXrqI82sLPA9wZyIkeXnTpCDy26zVnpG1QC1lYFS4m7P76qzGLB3Mj0CPrdSoNRu/x9Dvt1Z2JfcwZMNh0QE+bnNisUMM05GANqSS+obMGZ3gTBO5oo0ERGVox0eBkIvJQ0d67rBuNMwJLoHzdSA0qU/OU7Qfce8J2kbLV/S7bv0wMz+PUp9vLm2oH5GD3mIE24bZeaQUfTCrnDoPcKfyBgFyi0EoMFopdoBenWhjb1l85N9AaBoJVkwi1r1vjNpz2YzjoyiA8ralrz8XGNlSEGF2mtdxbUsLBUMNTH1mG4MSsTk+PQeRmaFkNFbr11LK+rfiM8ZYMTTLCHn6XVdoK3mfVVWc7zVuC7yKrSYmTgeuK3FTCisGKCpS6Cvlc1tKoZPzTljA6XP5xfQ+NtsoPFNf/6hc+lAwNXLg9xSW06An8FNc6vQi+ZabrTm0Vw6Q3CxkBZ9eNCnwVnW/AM6JvQQOTe8TvjCrGy4lCCMCGzoBJCK4y/a9H/6ka9zGfoQYYUpT4xKfi+ub5DCui8kIXLolrElFN82DTxvxbgrMMFSGu2A9F51x8bxFUFWKHScsMqDuGo20WCzjk2IAgbHRS9Z9ooVthk4J6bxH49uJ/rS0aUTBP+GAvphhHzJvv9Ad2E0Ocg5lm02Hop+GnLpgZYYg1wwz7pqOdxCbNk/uu3mmphpbnayITCJQexKWVCyhs2SF3adZXw5CGRu3fz918nWOkR4wImm4h0Vz/0yMwGCSm+d7a870iV0XG23WVnSYs4YOpaZvDfRFL4+yhjuFR41ywqdf3ruE4zrbR0jrNnM/ljRsNO6QxM1p2HCzIVGxDHoE3Z8qIL/qkC1PSV3leip29LTNuLaO1C+LK/35voDWwD3G9lTzigTdONU+jYNpjjsNvvLNRTrh/bR91+/NcnwXwLFbq4OzLKkv+cNawpHP3ZEospvGgQdOF6TdcvvplgtiWfJWoiQSoXlmsipj/7chxgPv86fIlj1KFEuDDhxQkYnRa2h3i5wnI9bLhiE0+dvAducKNZfx8t9nW2liqdJN03JZiNpJ52Wki94pFLDfo0ja3Z/JFhwvMCa6B5/LFT7C/fzbM5ENcBbvz1U2d0Xr0i/rRqPMK3VlgYoCjQRlpy4HtErT9nGZ98e/uxN4hp2GjmmJHyDj9A6CR9+7F9a37PhXKnkUQx6goIWzGSO2SlNVPm5gOppp6OEtO2gBLFk5Ka6SjyAZqDQ8iHR3jnsdUZTiP3wu79Qah5IKcLYYJ2xrbfH5xFDoNpVIFChSVbMB2w9U/McKx15wys/121mEJIQfyvLCCK9+FXdv0p2/Scgir0WmMTW4biaVbkzxbsME8NztKGq9Mo7u3C7lze45eSPB12/XYsb74biNLShzmv9Rjvd/iTqfL/AwAA//+RqPFS" + return "eJzsXd1z2ziSf89fgZqXJFceze3c1T2krrbKcXZmUxvPutbOVN0TBwRbEiIQ4ODDjuavv8IXCUqURPFDjqfWeYktEv3rD3Q3Gg3oe7SB7Tu0ItUrhDTVDN6h1z8LsWKAbpgwBbpjWC+FLF+/QkgCA6zgHcpB41cIFaCIpJWmgr9Df32FEEI/39yhUhSGwSuElhRYod65D75HHJcQSdkfva3s71KY+Jf0+fQdhnNgqv5zfFXkX4Do5M8deOKPx8WpFpLyFSpBS0rU/si7EFIYRoFc/Efro4NQ7I//Y+af2MD2Sciic+ASNC6wxnMNblmdZWy1VRrKWYaWoISRBCYbPA78XS0Q/++703bVGrcQJnfW3fFpVuKqonwVHv2uNfgR67wN5qjXWCMJ2kgOBVpKUaLWZLy++4h+NyC3iz22csoY5atD9FrDvPfPRtNI3tmd4W3BpHMVdU6WiIYI5SXyal91XVpvYb0RSrtnFaKcMFMAkrAyDMsrpPHXK4SLL0bpEri+QpgXSArDCyt2kFLIRQceyh8FJZCVguv1EExRZBIqITVy43QRqqRw1kCLIVTu/Nvo4wcklkivIao10s2BCb5SSIsu4lpozDroLpnA+jDVB/taTQmXwnC9b2BElJXR0MvAbvyzMxrYkkp4wowtCimqCopFvtWgOni38jrM+kdORGm5dq+jMBjKt074kUgP+lmFyQa0yoiT3iNmpsv2e6IJg52Bh3KlMSewIJVZSFAgH6HIiJCgDoLZ82Q7cH4xZQ7SWqIbB8VhkeAOztrO02Cnkf4paEbhFWSalrBQQAaA+mwHQEshEWYsAKMcKSCCF6oX+UW1Ezn6UX6wGpCY2N8i25gxQbCGAt3cffbumypEjJTANdtaZEZBFFgfIRVUbRYS8FCLvrH2Z+F5i7Yj+UBiB+5FOBPVODOuIdghPYKP/0SiAontA0eV5FA8SaphGv7tUBo40qKfABzpiSXgxuwvghJKIbeL3NqW4AuJy0zRP2AgFGu1Li4Ex25ReQrWOK1V/nq7QA9rqoK3tgYsONsi/Igpwznzs+3X25Ce+BBkBWpfhh/REpeUbbuj7mGWjIJiIEu3Hn4zy+xYz8eNesJVRvlAe7X62dOMmzOUB1QrA0r7SUy1QuKJI0sTqQoTeBZuhdFTshsnqWOx4ViL5+CXg34ScrOgfCVBqancMAH6GHN6CyaQOQdJyAoWzjMNRxSTi1GYYELhKLC/PYIcCGJiuZwNx1RH8pnjxJsUKyQwwYehNVYoB+BIGs4pXx01WQ8gc25+EIy/MVxZH2qHQYpyAhHHE1ZIaSw1FFdJnrVA97isGBQIHkFu0f/8Z/PJ9VKDRMp+TvnqChVYYztRudDokSoaZ6mp7MT8y4/Nq3trDvtqJQXptej4EB6eYdWRLLOxJuuFqrA1v69AjBayy/IOyn0v5S+oTSC94nltDu8tHXRv6aCGTickwozSIBfrYqkWVmJcFDAHpr9/+OneSfkXS8B7WCwhmqi1R1oAwjWi03CVFtIuCAiuMKF625ERHEnPD+KOw9WofaGqRip4hGFt+uf3ZyA1mjL6h8viRoG1gbACSYBru6CpgXoyO8lND3yGrwEzvd5mORNkM4f+axLIk4gKt0/1UvkXkS+WmDIoZkD3ReTBJtf4EZCnYzXd0x4tuOhs50SXzpjz0CmTlzZRuoD0XOip6Vk/3RNnvcSZT9HNKmqMuhug8yl9F+kw1TdI5zSATqkONoMtlnxRF0cyn+dnjb+b2HX+3/W/fqlXtaqpyvQBWVVz+EpMNH0EDwxXFbMPWeH2QEQE15hymCWvcIASCqfhBN0dKEEMic4WUaoxO3SfOOzg2OSmxByvZpSPzW9uA43d7Ka/+VfgNyWmEqA1+5AWNHWC3+1K+cpL76ouQAbSdsLmgJaGLSljTR1ZkTUUhvXi4pFKbTALpeTpBR7Gbyq5VgOnYyERdvlhh/bF5MOCPb1TuKZKi5XE5Snpu+WRj5RCbKx0Awxo1tDuGfsfo+wqyPlOhbB9LdYh3CP1asXv7ChEfek4DHjCdVkRFEaeSkNn4d1FBlVZM6Q2iK3oo40SGmvoG8ieT3tpmDtbh/XLE2qyEcoz6DORRi+tprtfdqXSbzfwp/j0rCvzQhBTAteLAqzYR7iqh5aDUoYQUGppWE0CeRIHomcNxO3izAnDElDebH83ICkoJCRiQmxMdQqc32OZE52j0FHUWW0Oms3r31Yb+C3mJz5UlGlvxJMNXhJXdXfEzR2615hsCkkfQboGifC2a3DY7/hZCune+vkff3s9oR02u+QhtXLbjTaoZaO3PG9MaRh2+eTN3WfkxnPrh3oD1G2AhOBeI+hXrjvJA6Ml1SN3ki1sj9SNFndPG0Kj8aVlmem2dy1uj7jv1m7YnnCCQgRzLjSCrwSgQH9BWAXltT+w7zsq01VVf/zvMyQYssjJugUabYeRO4xz5gLyKVbnMZeG3XO6AXZNxqbsKwlYux0IzHcMJ7WaQPAZ7AaqNZQgMctCxdLPw4EbQp8EwQzVY9ZVUD/3KPcbRUO9xD7WKLdp0YZRZ8DrasDTgvX2NBZq2Bweo/uwm98dGCYFONOsL1MGXlic6O08gxwrm8ossWF64G5rEyUq18dlh1JXKJdiAxwV4om7ULGt4AqV+IuQrsGzpLy7r3MP4LiZfdsqqwyyxpkVcKkAFoz6Tx3DgkhHONdgL9P4UrtIxnL8xLIrDbVjuGvs2jf9Rv4zWO/RFomTlkpLSBYz3ij3+Wu1S5ylAy4K3x4aNg5wzmCybDgZM8mMw/xxlMdjnccb7CLv7wwuNJVrWVx0nZ2obvASu0buenZG2prv3+cd669xVrafhyYWMXE66jvBUoubGjt1bTHZUsLQVtqfJKQ1Lz+gxcm6mZoStu/tGtgCvGMflwHuEQ/sEnWID1nJ1JYxw+rKA03LcgPRhhxl/MRLnFvI7WbEeIGANDJB7bPe6iB32ew/le6Y6dQkI2EC1Z3X0znc8cn0jBYaG4Zjy3Pm9TiwYbjByXcEW7dUJ3D3WolH4FfA9VzYtcRchUafieFXtMh84WL4GZYSf0V3/lzkP+9H2qrFM+L8SXvrK/aHVFIQUCqeQxkLMiswlII/2zaS0z2DR2CxedYDGpXyRqYmXnnHtpZdnAOW45Uo9rzFeH92yk9Uohg5z1Lc1ktMj7nDP0wD+1EwU06RLzaI3bG7sKSoTwaFng1L8lIBPGFvhK3/0sVXOlePs3QC1yyZmh+8IzvLwfrKFPyxDO14FXGnNB5OOdb6DhjCqbGLnFdhAhc5ZpiTvlcxfBK4QO/jKzMemF9rXalFjskGeJGNK5O3A2ByiAvXW76hLeXvDw93P9w7uSAvGKtKgQKOTuPsRjos39kJ1RFbOMeeb2sg7gBGB9g+AFUluBq6LjsuSz90EGaN9Y2QiGCyhrdWlvBVg+SYOfxv7t/2ZeBSNkAYBa6VhXqehGdW/blgLqXmQ3MmyLELIvuvRbSCcSczD8J0yGpLe7i5++Hzh7sY83ewBjttMMeosGTiaYF+EtIO4H5TiOrXCjkNN9dw2KS06ZlHSkvApTvo24/5bNyJ0LYQWsdCpxPDCUbGnT4+rsaAJnUdvTiZV3WB45l1N5j3bvSUzz/rPn5632FLb5YjdPG2Hzsz66KbsdOTpEZ5mUmSwpxX6peeAglnnddJkSpTimWVFF+3C8KEclcKcQ7En2UaVj5JrxWqx4rtvRKQBllS7i7VcYtLOz/v7z8hD+MkzlEzcbdS2kjt19vERo0KFZ8+gMZZ6WFEjR5/vT0PEYenC+iRuJXbMCWKCvgEEG/CcankmKPRdrnpTke1YUthVmvnek5ibS8AGNbACT2ygdtxbKPHoY0WJ9d27a8lzU26zPakt4hgRgxz0nam8bQGnjbX+GscrCRixcCyFisi0blZN8aSj/cvCKm9taOttLNJn16H7PigtOo4L2FFhf3Pv+U3Tn6+deClSG+v5OlEtCuyldDo+uYfLQcnuJdVlJET2mFBLaXg2tqV9ShSH95zmE0u/3p4QCVgZaSViJAIMFkn3gbloJ8AeGQQ8+KUr6lThJc8aSIT3Svc2aYUOi3Vlz2V+sh1hon2soXWqppdQmjpWldq/ZyCc/flIi1p5Y+BBkFeNV4r5mytHMm6spboOlfMpxY5z8h7yzEfZLGeTrEw0DYNt/I7ncE+fxhSpRB6DYVj+w3lqFRvG/bTMPxaOUEojcnmykerknKjobWSZXgLMqxCKqxCtbJ2234+pHsih7Yu3JXz+zsd3TsaQy4ZPnSPfGVyZfJeg9+Z/N7kM+7HKI4rtRbaRXQmVqN2Pl3Mc9do1GdSlMIrt9PsOqQLf6i7JtoDkO8zyfJt5jPOiwLcOxHitRKQHENPBF/SVWaqAk/SJkPi9YB+4HBGH5E15itQV17jfj4l1wNsK/A3k4My7Li4uSmzKI3RK/Exek+BJGqfGdJwTQtWgNIRcoZXw65gvF6B845hU/dttFA/fIQ/RKD7ABOxXhzq+YI2eU17gckmMjLhnKptA5MNF08MipWfStfN7/XWXWuuFcCo28m3pE+in8XHGt5CXfPyBi82C7xAgWj9wdugjxTYSeBbDRkRg3dpW0L3J9qaOz+SfK9plnJNGDT0l2gRH0G/G6ExSrtFTmG/rCduFtopiBH+OWWlAFxkDLQGOecsqEzOqFp7wVuayNNEWlSUtHjpCbwURWanrh2MUQ5zon9aCwUoUnKLLq97B/hWFHS5vSabD/GBCeb1Ifay9KqhqRjd5yA63HRKTaCjcX0P/aB3Cr8nYJsoJHX0GbKXYAXxysGU+ms7TdQaAS8qQbkNa0a7Xq0t6FYc6cWH4TWt6fi4RGQIyUWM9daCGhbmSoa62NpNNyZlYnQeOi1D43KowK2zjvl11W2Eh2xwhAl28DO32g7wNrG+KsNYliS+s0SVhJGe8cR14WBUwJJyGgs+Xa8qSKoZPzTljBaXP5yOoeGrUHqLa/r4hU8tB3pGLgdxTm1aAn8GNU6vQieZcbpT61lwqTXCWkNZdeNCnzmjG3AMqCtfNLXvuEZSiWhZMSiBa7+yKAT4znH3RQH+Ask6r0D3wi9R6gtGONs2t90JDq0XFv6m44SYtIr3PUbuq+6sdYj6ssb0XXfLBq4qwBKVhmlaMX/nY2fpuiXodjSeds3cnReNyCB2UodZa5bngh8XfRTAHK7UDnvU3k/j4nrOZWTcMW67PC0iCptjh1boEStH35ZnVy8XifL3kdyddYN/npjfIcYZTDYSaQcR9ET1GnHBv7e2vG1JlRbDbLvNzpwWscPUpczgf4ko4K+DjOFc4V2yNtOeX/vljnEVmlOcPZvJ7zPqNxwnYLJun5mRqXDsZQjauOicNeAfquOMj/SuoDu+uj+upJ8U81313JX4u1sbHNyXsrXq0UZ3YxV6uPl6gu6LDzu9F6GKX/tlB8BVMxyItwcRv4RqWNiJGGTu32a9aAxH39rqYwwvCngxc22kPS9eTMabyGeO+oOXypmlkYDpW4/ZHubzB+thE2OvA+IifYu78QSTDaoJW3ZKyhitry948L6h7oalqvlqktho7jpi25NEuZPkuJ5S6UZbs8R2kbbj7dBNqxDmXb0aoVUU1z0pZ1X3nkfMrs4W8Tfydo2TlEgRQ8tRy6958/fAvTpAu9VqeB/ujJuv1xBXdORh+g/ANG68gvuaDezqoYkwrNnZT0rQa1E42jFT863Rdu3ZZQnY6PUfC0xYlmMFRRZUid13imSLKRDXKyhnlV5tzjXwYDjhZpKVxFxDgTxtpAQDtkWFcRMkPHl986kzS27YaOLTQHl/VuGb7a5vPqXf0tNxT/ZhJEGMqgJCl5RkFllp9Ji4viPV2HtT4iIVUKR4UFIT3bC0g2bnbqXdm4mmsdUJLlnqhN35rdPTQI73XHrtDD1VvHPFkh9MoQokyg3ZgG6Brb/HlmGlWlf7uC9vjz34nLivckQF3l45ZtzlQfE5CZU/Qop1aO0Kl/T4Hv1HzOK5TWH8jdAF7jxE1bqT1KUhWYyfL8fyprtYNaZNmLFaj+Haum9Mlf8fAAD//53HQcs=" } diff --git a/x-pack/metricbeat/module/gcp/loadbalancing/_meta/fields.yml b/x-pack/metricbeat/module/gcp/loadbalancing/_meta/fields.yml index 738e025bd15..054a71c1fd1 100644 --- a/x-pack/metricbeat/module/gcp/loadbalancing/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/loadbalancing/_meta/fields.yml @@ -60,3 +60,43 @@ - name: tcp_ssl_proxy.open_connections.value type: long description: Current number of outstanding connections through the TCP/SSL proxy. + - name: https.backend_latencies.value + type: object + object_type: histogram + description: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + - name: https.external.regional.backend_latencies.value + type: object + object_type: histogram + description: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + - name: https.external.regional.total_latencies.value + type: object + object_type: histogram + description: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. + - name: https.frontend_tcp_rtt.value + type: object + object_type: histogram + description: A distribution of the RTT measured for each connection between client and proxy. + - name: https.internal.backend_latencies.value + type: object + object_type: histogram + description: A distribution of the latency calculated from when the request was sent by the internal HTTP/S load balancer proxy to the backend until the proxy received from the backend the last byte of response. + - name: https.internal.total_latencies.value + type: object + object_type: histogram + description: A distribution of the latency calculated from when the request was received by the internal HTTP/S load balancer proxy until the proxy got ACK from client on last response byte. + - name: https.total_latencies.value + type: object + object_type: histogram + description: A distribution of the latency calculated from when the request was received by the external HTTP/S load balancer proxy until the proxy got ACK from client on last response byte. + - name: l3.external.rtt_latencies.value + type: object + object_type: histogram + description: A distribution of the round trip time latency, measured over TCP connections for the external network load balancer. + - name: l3.internal.rtt_latencies.value + type: object + object_type: histogram + description: A distribution of RTT measured over TCP connections for internal TCP/UDP load balancer flows. + - name: tcp_ssl_proxy.frontend_tcp_rtt.value + type: object + object_type: histogram + description: A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client. \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/loadbalancing/manifest.yml b/x-pack/metricbeat/module/gcp/loadbalancing/manifest.yml index f5e471af72f..2388f17bc13 100644 --- a/x-pack/metricbeat/module/gcp/loadbalancing/manifest.yml +++ b/x-pack/metricbeat/module/gcp/loadbalancing/manifest.yml @@ -25,3 +25,13 @@ input: - "tcp_ssl_proxy/ingress_bytes_count" - "tcp_ssl_proxy/new_connections" - "tcp_ssl_proxy/open_connections" + - "https/backend_latencies" + - "https/external/regional/backend_latencies" + - "https/external/regional/total_latencies" + - "https/frontend_tcp_rtt" + - "https/internal/backend_latencies" + - "https/internal/total_latencies" + - "https/total_latencies" + - "l3/external/rtt_latencies" + - "l3/internal/rtt_latencies" + - "tcp_ssl_proxy/frontend_tcp_rtt" \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/metrics/response_parser.go b/x-pack/metricbeat/module/gcp/metrics/response_parser.go index a51e89ed17c..c31947c8434 100644 --- a/x-pack/metricbeat/module/gcp/metrics/response_parser.go +++ b/x-pack/metricbeat/module/gcp/metrics/response_parser.go @@ -130,22 +130,32 @@ var reMapping = map[string]string{ "pod.volume.utilization.value": "pod.volume.utilization.pct", // gcp.loadbalancing metricset - "https.backend_request_bytes_count.value": "https.backend_request.bytes", - "https.backend_request_count.value": "https.backend_request.count", - "https.backend_response_bytes_count.value": "https.backend_response.bytes", - "https.request_bytes_count.value": "https.request.bytes", - "https.request_count.value": "https.request.count", - "https.response_bytes_count.value": "https.response.bytes", - "l3.external.egress_bytes_count.value": "l3.external.egress.bytes", - "l3.external.egress_packets_count.value": "l3.external.egress_packets.count", - "l3.external.ingress_bytes_count.value": "l3.external.ingress.bytes", - "l3.external.ingress_packets_count.value": "l3.external.ingress_packets.count", - "l3.internal.egress_bytes_count.value": "l3.internal.egress.bytes", - "l3.internal.egress_packets_count.value": "l3.internal.egress_packets.count", - "l3.internal.ingress_bytes_count.value": "l3.internal.ingress.bytes", - "l3.internal.ingress_packets_count.value": "l3.internal.ingress_packets.count", - "tcp_ssl_proxy.egress_bytes_count.value": "tcp_ssl_proxy.egress.bytes", - "tcp_ssl_proxy.ingress_bytes_count.value": "tcp_ssl_proxy.ingress.bytes", + "https.backend_request_bytes_count.value": "https.backend_request.bytes", + "https.backend_request_count.value": "https.backend_request.count", + "https.backend_response_bytes_count.value": "https.backend_response.bytes", + "https.request_bytes_count.value": "https.request.bytes", + "https.request_count.value": "https.request.count", + "https.response_bytes_count.value": "https.response.bytes", + "l3.external.egress_bytes_count.value": "l3.external.egress.bytes", + "l3.external.egress_packets_count.value": "l3.external.egress_packets.count", + "l3.external.ingress_bytes_count.value": "l3.external.ingress.bytes", + "l3.external.ingress_packets_count.value": "l3.external.ingress_packets.count", + "l3.internal.egress_bytes_count.value": "l3.internal.egress.bytes", + "l3.internal.egress_packets_count.value": "l3.internal.egress_packets.count", + "l3.internal.ingress_bytes_count.value": "l3.internal.ingress.bytes", + "l3.internal.ingress_packets_count.value": "l3.internal.ingress_packets.count", + "tcp_ssl_proxy.egress_bytes_count.value": "tcp_ssl_proxy.egress.bytes", + "tcp_ssl_proxy.ingress_bytes_count.value": "tcp_ssl_proxy.ingress.bytes", + "https.backend_latencies.value": "https.backend_latencies.value", + "https.external.regional.backend_latencies.value": "https.external.regional.backend_latencies.value", + "https.external.regional.total_latencies.value": "https.external.regional.total_latencies.value", + "https.frontend_tcp_rtt.value": "https.frontend_tcp_rtt.value", + "https.internal.backend_latencies.value": "https.internal.backend_latencies.value", + "https.internal.total_latencies.value": "https.internal.total_latencies.value", + "https.total_latencies.value": "https.total_latencies.value", + "l3.external.rtt_latencies.value": "l3.external.rtt_latencies.value", + "l3.internal.rtt_latencies.value": "l3.internal.rtt_latencies.value", + "tcp_ssl_proxy.frontend_tcp_rtt.value": "tcp_ssl_proxy.frontend_tcp_rtt.value", // gcp.metrics metricset // NOTE: nothing here; if the user directly uses this metricset the mapping to ECS is @@ -198,6 +208,8 @@ var reMapping = map[string]string{ "topic.send_request_count.value": "topic.send_request.count", "topic.streaming_pull_response_count.value": "topic.streaming_pull_response.count", "topic.unacked_bytes_by_region.value": "topic.unacked_bytes_by_region.bytes", + "subscription.ack_latencies.value": "subscription.ack_latencies.value", + "subscription.push_request_latencies.value": "subscription.push_request_latencies.value", // gcp.storage metricset "api.request_count.value": "api.request.count", @@ -233,6 +245,10 @@ var reMapping = map[string]string{ "cluster.yarn.nodemanagers.value": "cluster.yarn.nodemanagers.count", "cluster.yarn.pending_memory_size.value": "cluster.yarn.pending_memory_size.value", "cluster.yarn.virtual_cores.value": "cluster.yarn.virtual_cores.count", + "cluster.job.completion_time.value": "cluster.job.completion_time.value", + "cluster.job.duration.value": "cluster.job.duration.value", + "cluster.operation.completion_time.value": "cluster.operation.completion_time.value", + "cluster.operation.duration.value": "cluster.operation.duration.value", } func remap(l *logp.Logger, s string) string { @@ -259,7 +275,11 @@ func getValueFromPoint(p *monitoring.Point) (out interface{}) { out = v.StringValue case *monitoring.TypedValue_DistributionValue: // Distribution values aren't simple values. Take a look at this - out = v.DistributionValue + histogram := gcp.DistributionHistogramToES(v.DistributionValue) + + out = mapstr.M{ + "histogram": histogram, + } } return out diff --git a/x-pack/metricbeat/module/gcp/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/gcp/pubsub/_meta/fields.yml index 4c696e6628c..bde92cfe059 100644 --- a/x-pack/metricbeat/module/gcp/pubsub/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/pubsub/_meta/fields.yml @@ -81,9 +81,6 @@ - name: subscription.push_request.count type: long description: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. - - name: subscription.push_request_latencies.value - type: long - description: Distribution of push request latencies (in microseconds), grouped by result. - name: subscription.retained_acked.bytes type: long description: Total byte size of the acknowledged messages retained in a subscription. @@ -124,7 +121,8 @@ type: long description: Cumulative count of configuration changes, grouped by operation type and result. - name: topic.message_sizes.bytes - type: long + type: object + object_type: histogram description: Distribution of publish message sizes (in bytes) - name: topic.oldest_retained_acked_message_age_by_region.value type: long @@ -147,3 +145,11 @@ - name: topic.unacked_bytes_by_region.bytes type: long description: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. + - name: subscription.ack_latencies.value + type: object + object_type: histogram + description: Distribution of ack latencies in milliseconds. The ack latency is the time between when Cloud Pub/Sub sends a message to a subscriber client and when Cloud Pub/Sub receives an Acknowledge request for that message. + - name: subscription.push_request_latencies.value + type: object + object_type: histogram + description: Distribution of push request latencies (in microseconds), grouped by result. \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/pubsub/manifest.yml b/x-pack/metricbeat/module/gcp/pubsub/manifest.yml index 12deb4910be..887909ebe9f 100644 --- a/x-pack/metricbeat/module/gcp/pubsub/manifest.yml +++ b/x-pack/metricbeat/module/gcp/pubsub/manifest.yml @@ -51,3 +51,7 @@ input: - "topic/send_message_operation_count" - "topic/send_request_count" - "topic/unacked_bytes_by_region" + - "subscription/ack_latencies" + - "subscription/push_request_latencies" + - "topic/message_sizes" + diff --git a/x-pack/osquerybeat/Jenkinsfile.yml b/x-pack/osquerybeat/Jenkinsfile.yml index dce1d4a383d..bd3c42fa54b 100644 --- a/x-pack/osquerybeat/Jenkinsfile.yml +++ b/x-pack/osquerybeat/Jenkinsfile.yml @@ -13,12 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/osquerybeat check; - make -C x-pack/osquerybeat update; - make check-no-changes; - stage: checks unitTest: mage: "mage build unitTest" stage: mandatory diff --git a/x-pack/osquerybeat/internal/config/config.go b/x-pack/osquerybeat/internal/config/config.go index c80d64603bc..0d23af8186f 100644 --- a/x-pack/osquerybeat/internal/config/config.go +++ b/x-pack/osquerybeat/internal/config/config.go @@ -8,6 +8,8 @@ package config import ( + "fmt" + "github.com/elastic/beats/v7/libbeat/processors" ) @@ -19,9 +21,13 @@ import ( // type: logs // query: select * from usb_devices -const DefaultNamespace = "default" +const ( + DefaultNamespace = "default" + DefaultDataset = "osquery_manager.result" + DefaultType = "logs" +) -const datastreamPrefix = "logs-osquery_manager.result-" +var datastreamPrefix = fmt.Sprintf("%s-%s-", DefaultType, DefaultDataset) type StreamConfig struct { ID string `config:"id"` @@ -34,6 +40,8 @@ type StreamConfig struct { type DatastreamConfig struct { Namespace string `config:"namespace"` + Dataset string `config:"dataset"` + Type string `config:"type"` } type InputConfig struct { diff --git a/x-pack/osquerybeat/internal/distro/distro.go b/x-pack/osquerybeat/internal/distro/distro.go index 9294c24022d..9fc01943a26 100644 --- a/x-pack/osquerybeat/internal/distro/distro.go +++ b/x-pack/osquerybeat/internal/distro/distro.go @@ -36,14 +36,14 @@ const ( osqueryCertsDarwinPath = "private/var/osquery/certs/" + osqueryCertsPEM osqueryCertsWindowsPath = "osquery/certs/" + osqueryCertsPEM - osqueryVersion = "5.2.2" + osqueryVersion = "5.4.0" osqueryMSIExt = ".msi" osqueryPkgExt = ".pkg" - osqueryDistroDarwinSHA256 = "c1db00554f65a1f240e9c827c73e0a768fbda66475b18bd68786b3a12e04200f" - osqueryDistroLinuxSHA256 = "e86e4cec2f941782a6223a09c2e9d7bdc6cfea0e30ba9792056749b0e79f4576" - osqueryDistroLinuxARMSHA256 = "799f4851adeafd251aa57a91e20a9180c5b0c5e0d06cfc12815a1eaf631aaaa1" - osqueryDistroWindowsSHA256 = "d784b9c114ae2f5216dc5aa6bf311863c2db8fdaca31085e38a51b35eefa6c50" + osqueryDistroDarwinSHA256 = "82d00dad86c388b0f9c13a6b3220de8cefde504c609f43167f34ab98ec19fbda" + osqueryDistroLinuxSHA256 = "6616a2de6f4a54a7454642aed88f5e78d4e8d22267c1b943500ca4c4c610fc8d" + osqueryDistroLinuxARMSHA256 = "ac10b9c87b6dad7b1a4611d82ce09f4c11e94f58494616579328416d6e05264f" + osqueryDistroWindowsSHA256 = "7237c7acc69049b63841f16f64516a525bceaf5d16c7ff6b383c041cbee21154" ) type OSArch struct { diff --git a/x-pack/osquerybeat/internal/pub/publisher.go b/x-pack/osquerybeat/internal/pub/publisher.go index be4bfd555c7..dfe5fad7850 100644 --- a/x-pack/osquerybeat/internal/pub/publisher.go +++ b/x-pack/osquerybeat/internal/pub/publisher.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/add_data_stream" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/config" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/ecs" "github.com/elastic/elastic-agent-libs/logp" @@ -44,7 +45,7 @@ func (p *Publisher) Configure(inputs []config.InputConfig) error { p.mx.Lock() defer p.mx.Unlock() - processors, err := processorsForInputsConfig(inputs) + processors, err := p.processorsForInputsConfig(inputs) if err != nil { return err } @@ -90,20 +91,42 @@ func (p *Publisher) Close() { } } -func processorsForInputsConfig(inputs []config.InputConfig) (procs *processors.Processors, err error) { +func (p *Publisher) processorsForInputsConfig(inputs []config.InputConfig) (procs *processors.Processors, err error) { + procs = processors.NewList(nil) + // Use only first input processor // Every input will have a processor that adds the elastic_agent info, we need only one // Not expecting other processors at the moment and this needs to work for 7.13 for _, input := range inputs { if len(input.Processors) > 0 { - procs, err = processors.New(input.Processors) + // Attach the data_stream processor. This will append the data_stream attributes to the events. + // This is needed for the proper logstash auto-discovery of the destination datastream for the results. + ds := add_data_stream.DataStream{ + Namespace: input.Datastream.Namespace, + Dataset: input.Datastream.Dataset, + Type: input.Datastream.Type, + } + if ds.Namespace == "" { + ds.Namespace = config.DefaultNamespace + } + if ds.Dataset == "" { + ds.Dataset = config.DefaultDataset + } + if ds.Type == "" { + ds.Type = config.DefaultType + } + + procs.AddProcessor(add_data_stream.New(ds)) + + userProcs, err := processors.New(input.Processors) if err != nil { return nil, err } - return procs, nil + procs.AddProcessors(*userProcs) + break } } - return nil, nil + return procs, nil } func hitToEvent(index, eventType, actionID, responseID string, hit map[string]interface{}, ecsm ecs.Mapping, reqData interface{}) beat.Event { diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index 61ff7059c61..466ab053e65 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -13,15 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: - checks: - mage: | - mage check; - mage update; - make: | - make -C packetbeat check; - make -C packetbeat update; - make check-no-changes; - stage: checks arm: mage: "mage build unitTest" platforms: ## override default label in this specific stage. diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 546a21e6346..041dde58d20 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -13,16 +13,6 @@ when: tags: true ## for all the tags platform: "windows-2022" ## default label for all the stages stages: - checks: - make: | - make -C x-pack/winlogbeat check; - make -C x-pack/winlogbeat update; - make -C winlogbeat check; - make -C winlogbeat update; - make check-no-changes; - platforms: ## override default labels in this specific stage. - - "immutable && ubuntu-18" - stage: checks build: mage: "mage build unitTest" withModule: true @@ -49,12 +39,11 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2012-r2" stage: extended_win - # See https://github.com/elastic/beats/issues/32469 - # windows-11: - # mage: "mage build unitTest" - # platforms: ## override default labels in this specific stage. - # - "windows-11" - # stage: extended_win + windows-11: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-11" + stage: extended_win windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage.