From c73e77ac3fa35a2d717a46dccc22acb28aeb18b1 Mon Sep 17 00:00:00 2001 From: Oleksiy Kovyrin Date: Fri, 5 Nov 2021 14:49:29 -0400 Subject: [PATCH] Remove the now deprecated appsearch module from metricbeat (#28850) (cherry picked from commit 82f6f895c65e13ce29cf45610915ec4b58ff3cf4) --- metricbeat/docs/fields.asciidoc | 273 ------------------ metricbeat/docs/modules/appsearch.asciidoc | 49 ---- .../docs/modules/appsearch/stats.asciidoc | 25 -- metricbeat/docs/modules_list.asciidoc | 3 - x-pack/metricbeat/include/list.go | 2 - x-pack/metricbeat/metricbeat.reference.yml | 9 - .../module/appsearch/_meta/Dockerfile | 7 - .../module/appsearch/_meta/config.yml | 7 - .../_meta/docker-entrypoint-dependencies.sh | 10 - .../module/appsearch/_meta/docs.asciidoc | 6 - .../module/appsearch/_meta/fields.yml | 11 - x-pack/metricbeat/module/appsearch/doc.go | 6 - .../module/appsearch/docker-compose.yml | 27 -- x-pack/metricbeat/module/appsearch/fields.go | 23 -- .../module/appsearch/stats/_meta/data.json | 49 ---- .../appsearch/stats/_meta/docs.asciidoc | 1 - .../module/appsearch/stats/_meta/fields.yml | 120 -------- .../stats/_meta/test/required.stats.json | 65 ----- .../appsearch/stats/_meta/test/stats.json | 124 -------- .../metricbeat/module/appsearch/stats/data.go | 76 ----- .../module/appsearch/stats/data_test.go | 29 -- .../module/appsearch/stats/stats.go | 81 ------ .../appsearch/stats/stats_integration_test.go | 52 ---- .../module/appsearch/test_appsearch.py | 34 --- .../modules.d/appsearch.yml.disabled | 10 - 25 files changed, 1099 deletions(-) delete mode 100644 metricbeat/docs/modules/appsearch.asciidoc delete mode 100644 metricbeat/docs/modules/appsearch/stats.asciidoc delete mode 100644 x-pack/metricbeat/module/appsearch/_meta/Dockerfile delete mode 100644 x-pack/metricbeat/module/appsearch/_meta/config.yml delete mode 100755 x-pack/metricbeat/module/appsearch/_meta/docker-entrypoint-dependencies.sh delete mode 100644 x-pack/metricbeat/module/appsearch/_meta/docs.asciidoc delete mode 100644 x-pack/metricbeat/module/appsearch/_meta/fields.yml delete mode 100644 x-pack/metricbeat/module/appsearch/doc.go delete mode 100644 x-pack/metricbeat/module/appsearch/docker-compose.yml delete mode 100644 x-pack/metricbeat/module/appsearch/fields.go delete mode 100644 x-pack/metricbeat/module/appsearch/stats/_meta/data.json delete mode 100644 x-pack/metricbeat/module/appsearch/stats/_meta/docs.asciidoc delete mode 100644 x-pack/metricbeat/module/appsearch/stats/_meta/fields.yml delete mode 100644 x-pack/metricbeat/module/appsearch/stats/_meta/test/required.stats.json delete mode 100644 x-pack/metricbeat/module/appsearch/stats/_meta/test/stats.json delete mode 100644 x-pack/metricbeat/module/appsearch/stats/data.go delete mode 100644 x-pack/metricbeat/module/appsearch/stats/data_test.go delete mode 100644 x-pack/metricbeat/module/appsearch/stats/stats.go delete mode 100644 x-pack/metricbeat/module/appsearch/stats/stats_integration_test.go delete mode 100644 x-pack/metricbeat/module/appsearch/test_appsearch.py delete mode 100644 x-pack/metricbeat/modules.d/appsearch.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e991cfec84d..875f8672214 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16,7 +16,6 @@ grouped in the following categories: * <> * <> * <> -* <> * <> * <> * <> @@ -1465,278 +1464,6 @@ type: long Total. -type: long - --- - -[[exported-fields-appsearch]] -== App Search fields - -App Search module - - - -[float] -=== appsearch - - - - -[float] -=== stats - -App Search stats - - - -[float] -=== jvm - -JVM stats - - - -[float] -=== memory_usage - -Memory usage - - - -*`appsearch.stats.jvm.memory_usage.heap_init.bytes`*:: -+ --- -Heap init used by the JVM in bytes. - - -type: long - --- - -*`appsearch.stats.jvm.memory_usage.heap_used.bytes`*:: -+ --- -Heap used by the JVM in bytes. - - -type: long - --- - -*`appsearch.stats.jvm.memory_usage.heap_committed.bytes`*:: -+ --- -Committed heap to the JVM in bytes. - - -type: long - --- - -*`appsearch.stats.jvm.memory_usage.heap_max.bytes`*:: -+ --- -Max heap used by the JVM in bytes - - -type: long - --- - -*`appsearch.stats.jvm.memory_usage.non_heap_init.bytes`*:: -+ --- -Non-Heap initial memory used by the JVM in bytes. - - -type: long - --- - -*`appsearch.stats.jvm.memory_usage.non_heap_committed.bytes`*:: -+ --- -Non-Heap committed memory used by the JVM in bytes. - - -type: long - --- - -[float] -=== queues - -Worker queues - - - -*`appsearch.stats.queues.analytics_events.count`*:: -+ --- -Number of pending jobs in the `analytics_events` queue. - - -type: long - --- - -*`appsearch.stats.queues.document_destroyer.count`*:: -+ --- -Number of pending jobs in the `document_destroyer` queue. - - -type: long - --- - -*`appsearch.stats.queues.engine_destroyer.count`*:: -+ --- -Number of pending jobs in the `engine_destroyer` queue. - - -type: long - --- - -*`appsearch.stats.queues.index_adder.count`*:: -+ --- -Number of pending jobs in the `index_adder` queue. - - -type: long - --- - -*`appsearch.stats.queues.indexed_doc_remover.count`*:: -+ --- -Number of pending jobs in the `indexed_doc_remover` queue. - - -type: long - --- - -*`appsearch.stats.queues.mailer.count`*:: -+ --- -Number of pending jobs in the `mailer` queue. - - -type: long - --- - -*`appsearch.stats.queues.refresh_document_counts.count`*:: -+ --- -Number of pending jobs in the `refresh_document_counts` queue. - - -type: long - --- - -*`appsearch.stats.queues.reindexer.count`*:: -+ --- -Number of pending jobs in the `reindexer` queue. - - -type: long - --- - -*`appsearch.stats.queues.schema_updater.count`*:: -+ --- -Number of pending jobs in the `schema_updater` queue. - - -type: long - --- - -*`appsearch.stats.queues.failed.count`*:: -+ --- -Number of failed jobs waiting to be retried. - - -type: long - --- - -[float] -=== requests - -Request metrics - - - -*`appsearch.stats.requests.count`*:: -+ --- -Number of recently completed requests - - -type: long - --- - -[float] -=== api.duration - -API response time metrics - - - -*`appsearch.stats.requests.api.duration.avg.ms`*:: -+ --- -Average response time in milliseconds - - -type: long - --- - -*`appsearch.stats.requests.api.duration.max.ms`*:: -+ --- -Max response time in milliseconds - - -type: long - --- - -[float] -=== web.response_time - -Dashboard response time metrics - - - -*`appsearch.stats.requests.web.response_time.avg.ms`*:: -+ --- -Average response time in milliseconds - - -type: long - --- - -*`appsearch.stats.requests.web.response_time.max.ms`*:: -+ --- -Max response time in milliseconds - - type: long -- diff --git a/metricbeat/docs/modules/appsearch.asciidoc b/metricbeat/docs/modules/appsearch.asciidoc deleted file mode 100644 index c674f434095..00000000000 --- a/metricbeat/docs/modules/appsearch.asciidoc +++ /dev/null @@ -1,49 +0,0 @@ -//// -This file is generated! See scripts/mage/docs_collector.go -//// - -:modulename: appsearch - -[[metricbeat-module-appsearch]] -[role="xpack"] -== App Search module - -beta[] - -This is the App Search module. - -[NOTE] -===== -This module does not support collecting data from App Search running on {ecloud}. -===== - -[float] -=== Example configuration - -The App Search module supports the standard configuration options that are described -in <>. Here is an example configuration: - -[source,yaml] ----- -metricbeat.modules: -- module: appsearch - metricsets: ["stats"] - enabled: true - period: 10s - hosts: ["http://localhost:3002"] - #username: "elastic" - #password: "changeme" ----- - -This module supports TLS connections when using `ssl` config field, as described in <>. -It also supports the options described in <>. - -[float] -=== Metricsets - -The following metricsets are available: - -* <> - -include::appsearch/stats.asciidoc[] - diff --git a/metricbeat/docs/modules/appsearch/stats.asciidoc b/metricbeat/docs/modules/appsearch/stats.asciidoc deleted file mode 100644 index d2cbb6a5671..00000000000 --- a/metricbeat/docs/modules/appsearch/stats.asciidoc +++ /dev/null @@ -1,25 +0,0 @@ -//// -This file is generated! See scripts/mage/docs_collector.go -//// - -[[metricbeat-metricset-appsearch-stats]] -[role="xpack"] -=== App Search stats metricset - -beta[] - -include::../../../../x-pack/metricbeat/module/appsearch/stats/_meta/docs.asciidoc[] - -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. - -==== Fields - -For a description of each field in the metricset, see the -<> section. - -Here is an example document generated by this metricset: - -[source,json] ----- -include::../../../../x-pack/metricbeat/module/appsearch/stats/_meta/data.json[] ----- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index fb90a1e5e49..ad5d929514f 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -15,8 +15,6 @@ This file is generated! See scripts/mage/docs_collector.go .1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> -|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .17+| .17+| |<> beta[] |<> @@ -316,7 +314,6 @@ include::modules/activemq.asciidoc[] include::modules/aerospike.asciidoc[] include::modules/airflow.asciidoc[] include::modules/apache.asciidoc[] -include::modules/appsearch.asciidoc[] include::modules/aws.asciidoc[] include::modules/awsfargate.asciidoc[] include::modules/azure.asciidoc[] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index b14417a47cd..f06eed2652a 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -10,8 +10,6 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/airflow" - _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/appsearch" - _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/appsearch/stats" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/billing" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/cloudwatch" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 0905b027d39..aa21538d14b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -183,15 +183,6 @@ metricbeat.modules: # Password of hosts. Empty by default #password: password -#------------------------------ App Search Module ------------------------------ -- module: appsearch - metricsets: ["stats"] - enabled: true - period: 10s - hosts: ["http://localhost:3002"] - #username: "elastic" - #password: "changeme" - #--------------------------------- AWS Module --------------------------------- - module: aws period: 300s diff --git a/x-pack/metricbeat/module/appsearch/_meta/Dockerfile b/x-pack/metricbeat/module/appsearch/_meta/Dockerfile deleted file mode 100644 index c7e12eedd18..00000000000 --- a/x-pack/metricbeat/module/appsearch/_meta/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG APPSEARCH_VERSION -FROM docker.elastic.co/app-search/app-search:${APPSEARCH_VERSION} - -COPY docker-entrypoint-dependencies.sh /usr/local/bin/ -ENTRYPOINT /usr/local/bin/docker-entrypoint-dependencies.sh - -HEALTHCHECK --interval=1s --retries=300 --start-period=60s CMD python -c 'import urllib, json; response = urllib.urlopen("http://myelastic:changeme@localhost:3002/api/as/v1/internal/health"); response.getcode() == 200 or exit(1); data = json.loads(response.read()); exit(0);' diff --git a/x-pack/metricbeat/module/appsearch/_meta/config.yml b/x-pack/metricbeat/module/appsearch/_meta/config.yml deleted file mode 100644 index e3891aa53f3..00000000000 --- a/x-pack/metricbeat/module/appsearch/_meta/config.yml +++ /dev/null @@ -1,7 +0,0 @@ -- module: appsearch - metricsets: ["stats"] - enabled: true - period: 10s - hosts: ["http://localhost:3002"] - #username: "elastic" - #password: "changeme" diff --git a/x-pack/metricbeat/module/appsearch/_meta/docker-entrypoint-dependencies.sh b/x-pack/metricbeat/module/appsearch/_meta/docker-entrypoint-dependencies.sh deleted file mode 100755 index f95d8847872..00000000000 --- a/x-pack/metricbeat/module/appsearch/_meta/docker-entrypoint-dependencies.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -until curl -f -s "http://elasticsearch:9200/_license"; do - echo "Elasticsearch not available yet". - sleep 1 -done - -/usr/local/bin/docker-entrypoint.sh "$@" diff --git a/x-pack/metricbeat/module/appsearch/_meta/docs.asciidoc b/x-pack/metricbeat/module/appsearch/_meta/docs.asciidoc deleted file mode 100644 index fbb0016ce3c..00000000000 --- a/x-pack/metricbeat/module/appsearch/_meta/docs.asciidoc +++ /dev/null @@ -1,6 +0,0 @@ -This is the App Search module. - -[NOTE] -===== -This module does not support collecting data from App Search running on {ecloud}. -===== \ No newline at end of file diff --git a/x-pack/metricbeat/module/appsearch/_meta/fields.yml b/x-pack/metricbeat/module/appsearch/_meta/fields.yml deleted file mode 100644 index 5cf3be45b72..00000000000 --- a/x-pack/metricbeat/module/appsearch/_meta/fields.yml +++ /dev/null @@ -1,11 +0,0 @@ -- key: appsearch - title: "App Search" - release: beta - settings: ["ssl", "http"] - description: > - App Search module - fields: - - name: appsearch - type: group - description: > - fields: diff --git a/x-pack/metricbeat/module/appsearch/doc.go b/x-pack/metricbeat/module/appsearch/doc.go deleted file mode 100644 index 4cafeeff20d..00000000000 --- a/x-pack/metricbeat/module/appsearch/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// Package appsearch is a Metricbeat module that contains MetricSets. -package appsearch diff --git a/x-pack/metricbeat/module/appsearch/docker-compose.yml b/x-pack/metricbeat/module/appsearch/docker-compose.yml deleted file mode 100644 index 04d326dc3ba..00000000000 --- a/x-pack/metricbeat/module/appsearch/docker-compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: '2.3' - -services: - appsearch: - image: docker.elastic.co/integrations-ci/beats-appsearch:${APPSEARCH_VERSION:-7.6.2}-1 - build: - context: ./_meta - args: - APPSEARCH_VERSION: ${APPSEARCH_VERSION:-7.6.2} - depends_on: - - elasticsearch - environment: - - "elasticsearch.host=http://elasticsearch:9200" - - "allow_es_settings_modification=true" - - "JAVA_OPTS=-Xms2g -Xmx2g" - ports: - - 3002 - command: --process app-server - - elasticsearch: - extends: - file: ../../../metricbeat/docker-compose.yml - service: elasticsearch - environment: - - "bootstrap.memory_lock=true" - - "action.auto_create_index=.app-search-*-logs-*,-.app-search-*,+*" - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" diff --git a/x-pack/metricbeat/module/appsearch/fields.go b/x-pack/metricbeat/module/appsearch/fields.go deleted file mode 100644 index c05dbcbc911..00000000000 --- a/x-pack/metricbeat/module/appsearch/fields.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package appsearch - -import ( - "github.com/elastic/beats/v7/libbeat/asset" -) - -func init() { - if err := asset.SetFields("metricbeat", "appsearch", asset.ModuleFieldsPri, AssetAppsearch); err != nil { - panic(err) - } -} - -// AssetAppsearch returns asset data. -// This is the base64 encoded zlib format compressed contents of module/appsearch. -func AssetAppsearch() string { - return "eJzs171u2zAQB/DdT3Hw3PgBNBQI2qEtkKBogXYoCoUS/5aY8Csk5URvX1CKVVm2IseFraUcaevux/OZIq/oAXVCzFoP5vJyQRREkEhoeW0tfW8mlwsiBwnmkVCGwBZEHiEIXfiEfi29l8t3tCxDsMvfCyIOnzthgzA6ofcLIqK/wUgZXkksiNYCkvuk+fyKNFPYhcQRaouECmcq+zJzIPhuqH44H1jw3eyhcHEM17YdB1O1o7eeYY6hpe+536id+THRRPY4vvy42Us9lr5PUFDG1WnlWYG9L73mOcIUx00Tn8bij/H6xBLMpkKLsMrqgOECd6XS6GLkC0dg4/gEZilmo8qDU1ZTKNGUV2hqAKtpbXz0ctp/gOZGKRHChbQfttma5BTMKWTFni+CvWHPLXOsvK9StdHpZRv31uirrnkFky9/7RO7o/NftkO6RXRp37aMLf+xQrWHPXlz/WncA9zhoFMbLNNM1kHkPsUGOvhVbiodRrfakRIeUb7bSmVwZNZkobnQBd2bzMc6xZLdDR137XL2u2AL5yavFHRIOXxwpoabi74vmcRDF0JjfvrQMQkXmuM5ZZzPZ+4RjuOCp9zkqYMym5nZu5RJvmJCzidus08iHdYOvky7v0HDnW0jGeEcsYz2F5qt3B1gkurzEoqlleUszOfdVUyi17Gb+AWwbaLW+sREvPfFo1wGcghOgB9+JTs8VvB7F5STX8rf2nikYtL8ra/l85fJIYcOso5HGSsRjzIjJejfd8WKV47FDOe4jV1//UwO3hrtQUEojFSPjryasU2xUuc+E15v4FiBAVxoUkJK4ZEbzV8/jMcrw9mZ8b7wNuKW94RstX0yjU+e46f/yHyZGeb4/wY4E3O6Af4EAAD//2c12Ks=" -} diff --git a/x-pack/metricbeat/module/appsearch/stats/_meta/data.json b/x-pack/metricbeat/module/appsearch/stats/_meta/data.json deleted file mode 100644 index c21f4d5e578..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/_meta/data.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "appsearch": { - "stats": { - "jvm": { - "memory_usage": { - "heap_committed.bytes": 343932928, - "heap_init.bytes": 65011712, - "heap_max.bytes": 1908932608, - "heap_used.bytes": 263346304, - "non_heap_committed.bytes": 323395584, - "non_heap_init.bytes": 2555904 - } - }, - "queues": { - "analytics_events.count": 0, - "document_destroyer.count": 0, - "engine_destroyer.count": 0, - "failed.count": 0, - "index_adder.count": 0, - "indexed_doc_remover.count": 0, - "mailer.count": 0, - "refresh_document_counts.count": 0, - "reindexer.count": 0, - "schema_updater.count": 0 - }, - "requests": { - "count": 48, - "web.response_time": { - "avg.ms": 91.95176263650258, - "max.ms": 137.27807998657227 - } - } - } - }, - "event": { - "dataset": "appsearch.stats", - "duration": 115000, - "module": "appsearch" - }, - "metricset": { - "name": "stats", - "period": 10000 - }, - "service": { - "address": "localhost:3002", - "type": "appsearch" - } -} \ No newline at end of file diff --git a/x-pack/metricbeat/module/appsearch/stats/_meta/docs.asciidoc b/x-pack/metricbeat/module/appsearch/stats/_meta/docs.asciidoc deleted file mode 100644 index 34601038b6b..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/_meta/docs.asciidoc +++ /dev/null @@ -1 +0,0 @@ -This is the stats metricset of the App Search module. diff --git a/x-pack/metricbeat/module/appsearch/stats/_meta/fields.yml b/x-pack/metricbeat/module/appsearch/stats/_meta/fields.yml deleted file mode 100644 index ae5babadfdd..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/_meta/fields.yml +++ /dev/null @@ -1,120 +0,0 @@ -- name: stats - type: group - release: beta - description: > - App Search stats - fields: - - name: jvm - type: group - description: > - JVM stats - fields: - - name: memory_usage - type: group - description: > - Memory usage - fields: - - name: heap_init.bytes - type: long - description: > - Heap init used by the JVM in bytes. - - name: heap_used.bytes - type: long - description: > - Heap used by the JVM in bytes. - - name: heap_committed.bytes - type: long - description: > - Committed heap to the JVM in bytes. - - name: heap_max.bytes - type: long - description: > - Max heap used by the JVM in bytes - - name: non_heap_init.bytes - type: long - description: > - Non-Heap initial memory used by the JVM in bytes. - - name: non_heap_committed.bytes - type: long - description: > - Non-Heap committed memory used by the JVM in bytes. - - name: queues - type: group - description: > - Worker queues - fields: - - name: analytics_events.count - type: long - description: > - Number of pending jobs in the `analytics_events` queue. - - name: document_destroyer.count - type: long - description: > - Number of pending jobs in the `document_destroyer` queue. - - name: engine_destroyer.count - type: long - description: > - Number of pending jobs in the `engine_destroyer` queue. - - name: index_adder.count - type: long - description: > - Number of pending jobs in the `index_adder` queue. - - name: indexed_doc_remover.count - type: long - description: > - Number of pending jobs in the `indexed_doc_remover` queue. - - name: mailer.count - type: long - description: > - Number of pending jobs in the `mailer` queue. - - name: refresh_document_counts.count - type: long - description: > - Number of pending jobs in the `refresh_document_counts` queue. - - name: reindexer.count - type: long - description: > - Number of pending jobs in the `reindexer` queue. - - name: schema_updater.count - type: long - description: > - Number of pending jobs in the `schema_updater` queue. - - name: failed.count - type: long - description: > - Number of failed jobs waiting to be retried. - - name: requests - type: group - description: > - Request metrics - fields: - - name: count - type: long - description: > - Number of recently completed requests - - name: api.duration - type: group - description: > - API response time metrics - fields: - - name: avg.ms - type: long - description: > - Average response time in milliseconds - - name: max.ms - type: long - description: > - Max response time in milliseconds - - name: web.response_time - type: group - description: > - Dashboard response time metrics - fields: - - name: avg.ms - type: long - description: > - Average response time in milliseconds - - name: max.ms - type: long - description: > - Max response time in milliseconds diff --git a/x-pack/metricbeat/module/appsearch/stats/_meta/test/required.stats.json b/x-pack/metricbeat/module/appsearch/stats/_meta/test/required.stats.json deleted file mode 100644 index d82c5f993e5..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/_meta/test/required.stats.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "jvm": { - "memory_usage": { - "heap_init": 65011712, - "heap_used": 542618440, - "heap_committed": 858783744, - "heap_max": 1908932608, - "non_heap_init": 2555904, - "non_heap_committed": 213164032 - }, - "memory_pools": [ - "Code Cache", - "Metaspace", - "Compressed Class Space", - "PS Eden Space", - "PS Survivor Space", - "PS Old Gen" - ], - "vm_version": "25.222-b10", - "vm_vendor": "Oracle Corporation", - "vm_name": "OpenJDK 64-Bit Server VM" - }, - "queues": { - "analytics_events": { - "pending": 0 - }, - "document_destroyer": { - "pending": 0 - }, - "engine_destroyer": { - "pending": 2 - }, - "index_adder": { - "pending": 45 - }, - "indexed_doc_remover": { - "pending": 0 - }, - "mailer": { - "pending": 1 - }, - "refresh_document_counts": { - "pending": 12 - }, - "reindexer": { - "pending": 0 - }, - "schema_updater": { - "pending": 0 - }, - "failed": [] - }, - "system_info": { - "java_version": "1.8.0_222", - "jruby_version": "9.2.5.0", - "os_name": "Linux", - "os_version": "4.9.93-linuxkit-aufs" - }, - "stats": null, - "version": { - "number": "7.3.2", - "build_hash": "001cc60f7d6b9a9085d808d312a1a7c08a6d88b9", - "build_date": "2019-09-11T19:21:20Z" - } -} diff --git a/x-pack/metricbeat/module/appsearch/stats/_meta/test/stats.json b/x-pack/metricbeat/module/appsearch/stats/_meta/test/stats.json deleted file mode 100644 index 64837ab7700..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/_meta/test/stats.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "jvm": { - "memory_usage": { - "heap_init": 268435456, - "heap_used": 894931464, - "heap_committed": 1276641280, - "heap_max": 3817865216, - "non_heap_init": 2555904, - "non_heap_committed": 88211456 - }, - "memory_pools": [ - "Code Cache", - "Metaspace", - "Compressed Class Space", - "PS Eden Space", - "PS Survivor Space", - "PS Old Gen" - ], - "vm_version": "25.172-b11", - "vm_vendor": "Oracle Corporation", - "vm_name": "Java HotSpot(TM) 64-Bit Server VM" - }, - "queues": { - "analytics_events": { - "pending": 0 - }, - "document_destroyer": { - "pending": 0 - }, - "engine_destroyer": { - "pending": 0 - }, - "index_adder": { - "pending": 1 - }, - "indexed_doc_remover": { - "pending": 0 - }, - "mailer": { - "pending": 0 - }, - "refresh_document_counts": { - "pending": 1 - }, - "reindexer": { - "pending": 0 - }, - "schema_updater": { - "pending": 0 - }, - "failed": [ - { - "_index": ".app-search-esqueues-me_queue_v1_failed", - "_type": "_doc", - "_id": "4c31b9340b0dbfd31cea37ae3e459bde30a31bac", - "_seq_no": 0, - "_primary_term": 1, - "_score": null, - "_source": { - "job_type": "ActiveJob::QueueAdapters::EsqueuesMeAdapter::JobWrapper", - "payload": { - "args": [ - { - "job_class": "Work::Engine::ReindexerWithProgress", - "job_id": "4c31b9340b0dbfd31cea37ae3e459bde30a31bac", - "queue_name": "reindexer", - "arguments": [ - "5d821fcbadd56b827710739f", - "5d8221e4add56be5678c89a8" - ], - "locale": "en", - "executions": 1 - } - ] - }, - "created_at": 1568809446422, - "failed_at": 1568809446421, - "error_message": "Cannot transition status via :enqueued from \"enqueued\" (Reason(s): Status cannot transition via \"enqueued\")", - "exception": "StateMachines::InvalidTransition", - "backtrace": [ - "a very long backtrace" - ] - } - } - ] - }, - "system_info": { - "java_version": "1.8.0_172", - "jruby_version": "9.2.5.0", - "os_name": "Mac OS X", - "os_version": "10.14.3" - }, - "stats": { - "pid": 94563, - "start": "2019-09-09T11:40:14+00:00", - "end": "2019-09-09T11:41:14+00:00", - "metrics": { - "timers.document_storage.elasticsearch.save_document!": { - "sum": 26248.596241726773, - "max": 353.8208259851672, - "mean": 45.64973259430743 - }, - "timers.actastic.relation.search": { - "sum": 3292.0772842189763, - "max": 208.33823998691514, - "mean": 12.469989712950667 - }, - "timers.api.request.duration": { - "sum": 117625.40864944458, - "max": 7016.742944717407, - "mean": 4056.0485741187786 - }, - "counters.all.request": 34, - "counters.api.request.success": 29, - "counters.web.request.success": 5, - "counters.web.request.redirect": 1 - } - }, - "version": { - "number": "7.3.2", - "build_hash": "001cc60f7d6b9a9085d808d312a1a7c08a6d88b9", - "build_date": "2019-09-11T19:21:20Z" - } -} diff --git a/x-pack/metricbeat/module/appsearch/stats/data.go b/x-pack/metricbeat/module/appsearch/stats/data.go deleted file mode 100644 index 6017da9695d..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/data.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package stats - -import ( - "encoding/json" - "errors" - - "github.com/elastic/beats/v7/libbeat/common" - s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" -) - -var ( - timerSchema = s.Schema{ - "max.ms": c.Float("max"), - "avg.ms": c.Float("mean"), - } - schema = s.Schema{ - "jvm": c.Dict("jvm", s.Schema{ - "memory_usage": c.Dict("memory_usage", s.Schema{ - "heap_init.bytes": c.Int("heap_init"), - "heap_used.bytes": c.Int("heap_used"), - "heap_committed.bytes": c.Int("heap_committed"), - "heap_max.bytes": c.Int("heap_max"), - "non_heap_init.bytes": c.Int("non_heap_init"), - "non_heap_committed.bytes": c.Int("non_heap_committed"), - }), - }), - "queues": c.Dict("queues", s.Schema{ - "analytics_events.count": c.Int("analytics_events.pending"), - "document_destroyer.count": c.Int("document_destroyer.pending"), - "engine_destroyer.count": c.Int("engine_destroyer.pending"), - "index_adder.count": c.Int("index_adder.pending"), - "indexed_doc_remover.count": c.Int("indexed_doc_remover.pending"), - "mailer.count": c.Int("mailer.pending"), - "refresh_document_counts.count": c.Int("refresh_document_counts.pending"), - "reindexer.count": c.Int("reindexer.pending"), - "schema_updater.count": c.Int("schema_updater.pending"), - "failed.count": c.Int("failed.count"), - }), - "requests": c.Dict( - "stats.metrics", - s.Schema{ - "api.response_time": c.Dict("timers.api.request.duration", timerSchema, c.DictOptional), - "web.response_time": c.Dict("timers.web.request.duration", timerSchema, c.DictOptional), - "count": c.Int("counters.all.request", s.Optional), - }, - c.DictOptional, - ), - } -) - -func eventMapping(input []byte) (common.MapStr, error) { - var data map[string]interface{} - err := json.Unmarshal(input, &data) - if err != nil { - return nil, err - } - - queues, ok := data["queues"].(map[string]interface{}) - if !ok { - return nil, errors.New("queues is not a map") - } - - failed, ok := queues["failed"].([]interface{}) - if !ok { - return nil, errors.New("queues.failed is not an array of maps") - } - queues["failed.count"] = len(failed) - - dataFields, err := schema.Apply(data) - return dataFields, err -} diff --git a/x-pack/metricbeat/module/appsearch/stats/data_test.go b/x-pack/metricbeat/module/appsearch/stats/data_test.go deleted file mode 100644 index 9ec726beaa8..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/data_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build !integration -// +build !integration - -package stats - -import ( - "io/ioutil" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestEventMapping(t *testing.T) { - files, err := filepath.Glob("./_meta/test/*.json") - assert.NoError(t, err) - - for _, f := range files { - input, err := ioutil.ReadFile(f) - assert.NoError(t, err) - - _, err = eventMapping(input) - assert.NoError(t, err, f) - } -} diff --git a/x-pack/metricbeat/module/appsearch/stats/stats.go b/x-pack/metricbeat/module/appsearch/stats/stats.go deleted file mode 100644 index be0742ee6f5..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/stats.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package stats - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/metricbeat/helper" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/mb/parse" -) - -const ( - // defaultScheme is the default scheme to use when it is not specified in - // the host config. - defaultScheme = "http" - - // defaultPath is the default path to the mod_status endpoint on the - // Apache HTTPD server. - defaultPath = "/api/as/v1/internal/health" -) - -var ( - hostParser = parse.URLHostParserBuilder{ - DefaultScheme: defaultScheme, - DefaultPath: defaultPath, - }.Build() -) - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("appsearch", "stats", New, - mb.WithHostParser(hostParser), - mb.DefaultMetricSet(), - ) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - mb.BaseMetricSet - http *helper.HTTP -} - -// New creates a new instance of the MetricSet. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) - if err != nil { - return nil, err - } - return &MetricSet{ - base, - http, - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right -// format. It publishes the event which is then forwarded to the output. In case -// of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { - content, err := m.http.FetchContent() - if err != nil { - return errors.Wrap(err, "error in fetch") - } - - event, err := eventMapping(content) - if err != nil { - return errors.Wrap(err, "error converting event") - } - - reporter.Event(mb.Event{MetricSetFields: event}) - - return nil -} diff --git a/x-pack/metricbeat/module/appsearch/stats/stats_integration_test.go b/x-pack/metricbeat/module/appsearch/stats/stats_integration_test.go deleted file mode 100644 index 9e56d150b2c..00000000000 --- a/x-pack/metricbeat/module/appsearch/stats/stats_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -//go:build integration -// +build integration - -package stats - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/tests/compose" - mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" -) - -func TestFetch(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 570, "appsearch") - - config := getConfig("stats", service.Host()) - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 errors, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) -} - -func TestData(t *testing.T) { - service := compose.EnsureUp(t, "appsearch") - - config := getConfig("stats", service.Host()) - - f := mbtest.NewReportingMetricSetV2Error(t, config) - err := mbtest.WriteEventsReporterV2Error(f, t, "") - if err != nil { - t.Fatal("write", err) - } -} - -// GetConfig returns config for App Search module -func getConfig(metricset string, host string) map[string]interface{} { - return map[string]interface{}{ - "module": "appsearch", - "metricsets": []string{metricset}, - "hosts": []string{host}, - } -} diff --git a/x-pack/metricbeat/module/appsearch/test_appsearch.py b/x-pack/metricbeat/module/appsearch/test_appsearch.py deleted file mode 100644 index 276c4e47f8f..00000000000 --- a/x-pack/metricbeat/module/appsearch/test_appsearch.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import sys -import unittest -from xpack_metricbeat import XPackTest, metricbeat - - -class Test(XPackTest): - COMPOSE_SERVICES = ['appsearch'] - COMPOSE_TIMEOUT = 600 - - @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, 'integration test') - def test_stats(self): - self.render_config_template(modules=[{ - "name": "appsearch", - "metricsets": ["stats"], - "hosts": [self.compose_host(service="appsearch")], - "period": "5s" - }]) - proc = self.start_beat(home=self.beat_path) - self.wait_until(lambda: self.output_lines() > 0) - proc.check_kill_and_wait() - self.assert_no_logged_warnings() - - output = self.read_output_json() - self.assertEqual(len(output), 1) - evt = output[0] - self.assert_fields_are_documented(evt) - - self.assertIn("appsearch", evt) - self.assertIn("stats", evt["appsearch"]) - - appsearch_stats = evt["appsearch"]["stats"] - self.assertIn("jvm", appsearch_stats) - self.assertIn("queues", appsearch_stats) diff --git a/x-pack/metricbeat/modules.d/appsearch.yml.disabled b/x-pack/metricbeat/modules.d/appsearch.yml.disabled deleted file mode 100644 index 16ad3fd73fc..00000000000 --- a/x-pack/metricbeat/modules.d/appsearch.yml.disabled +++ /dev/null @@ -1,10 +0,0 @@ -# Module: appsearch -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-appsearch.html - -- module: appsearch - metricsets: ["stats"] - enabled: true - period: 10s - hosts: ["http://localhost:3002"] - #username: "elastic" - #password: "changeme"