diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eeab7b1544ff..9841db769d1b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -89,6 +89,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Removes old module name aliases (gsuite) and removing old cyberark module in favor of the new cyberarkpas{pull}27915[27915] - Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526] - All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762] +- Remove deprecated fields in Kafka module. {pull}27938[27938] *Heartbeat* - Remove long deprecated `watch_poll` functionality. {pull}27166[27166] @@ -115,6 +116,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509] - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804] - Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933] +- Remove deprecated fields in Kafka module. {pull}27938[27938] *Packetbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 28165faae182..ce8fa78718b3 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -85190,24 +85190,6 @@ Kafka log lines. -*`kafka.log.level`*:: -+ --- -type: alias - -alias to: log.level - --- - -*`kafka.log.message`*:: -+ --- -type: alias - -alias to: message - --- - *`kafka.log.component`*:: + -- diff --git a/filebeat/module/kafka/fields.go b/filebeat/module/kafka/fields.go index cd26ca026f4d..42053910207d 100644 --- a/filebeat/module/kafka/fields.go +++ b/filebeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded zlib format compressed contents of module/kafka. func AssetKafka() string { - return "eJykkztywyAQhnud4h/31gFUpEmXTDpfYMdayYwQMLB24ttngPhF8GtMZ9Z8/6eFXWLifYeJhokaQJRo7rD4jL8XDdBzWHvlRFnT4a0BgFTDbPut5gYYFOs+dKm0hKGZT7i4ZO+4w+jt1v3tVJiXmHOUtuNxrwa7Cswry2o7QivDoT0rlokXqbxjfVE5ZJNWFIqKI9kk07Z2blajp+wmfsvVvJlDoJGfTKyfupZXDV7b2VnDRqrRE++/re+L2o12x/V+QEI2nFqvQsxRZsTg7dzWRTSF8itfkPigHWXmUxay8Uxl1Asaq8RL8Oc8PK3rr6F8+w9IAKuIgzJHhzgMbfHH2jzgzg3da89Ddv9uS7LujT7h7uyczIR/ytf9oBbwldlw5AV2ONm1zW8AAAD//95/TD8=" + return "eJyskkFugzAQRfec4iv7cAAW3XTXqrtcYAQDsTAeZE/acvvK0ATi0oSo9Q5/9P7TePZoeSjQUt1SBqhRywV2r/F7lwEVh9KbXo24Ak8ZAIwZOqlOljOgNmyrUIzRHo46nnHx6NBzgcbLqf++WWFeY5YoK83lbg32K3A6k6yVBtY4DvkiTBuXraV0vTh2epWe+1sePsRXSXbDIp7nMxJ65NHIhNhjXIPaS5evi1gK4f8kXuidJuZDFnr0TGnVHzQOI2+EP+bhqeRVjXQlNkgAh4iDcReHuCN58uPamuDOC90bzya7H6+lk+6NOS29Og6BmnRas5nyZ7rdG7WAt4mNnrxC6tkuz74CAAD//xD+FUk=" } diff --git a/filebeat/module/kafka/log/_meta/fields.yml b/filebeat/module/kafka/log/_meta/fields.yml index 32b9008f7483..a23ca5a6749e 100644 --- a/filebeat/module/kafka/log/_meta/fields.yml +++ b/filebeat/module/kafka/log/_meta/fields.yml @@ -3,15 +3,6 @@ description: > Kafka log lines. fields: - - name: level - type: alias - path: log.level - migration: true - - name: message - type: alias - path: message - migration: true - - name: component type: keyword description: > diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ac1ccfbcdb45..5edd1d4ed419 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -38022,33 +38022,6 @@ consumergroup -[float] -=== broker - -Broker Consumer Group Information have been read from (Broker handling the consumer group). - - - -*`kafka.consumergroup.broker.id`*:: -+ --- -Broker id - - -type: long - --- - -*`kafka.consumergroup.broker.address`*:: -+ --- -Broker address - - -type: keyword - --- - *`kafka.consumergroup.id`*:: + -- @@ -38058,30 +38031,6 @@ type: keyword -- -*`kafka.consumergroup.topic`*:: -+ --- - -deprecated:[6.5] - -Topic name - -type: keyword - --- - -*`kafka.consumergroup.partition`*:: -+ --- - -deprecated:[6.5] - -Partition ID - -type: long - --- - *`kafka.consumergroup.offset`*:: + -- @@ -38194,19 +38143,6 @@ Partition data. -*`kafka.partition.partition.id`*:: -+ --- - -deprecated:[6.5] - -Partition id. - - -type: long - --- - *`kafka.partition.partition.leader`*:: + -- @@ -38257,58 +38193,6 @@ type: long -- -*`kafka.partition.topic.error.code`*:: -+ --- - -deprecated:[6.5] - -topic error code. - - -type: long - --- - -*`kafka.partition.topic.name`*:: -+ --- - -deprecated:[6.5] - -Topic name - - -type: keyword - --- - -*`kafka.partition.broker.id`*:: -+ --- - -deprecated:[6.5] - -Broker id - - -type: long - --- - -*`kafka.partition.broker.address`*:: -+ --- - -deprecated:[6.5] - -Broker address - - -type: keyword - --- - [float] === producer diff --git a/metricbeat/module/kafka/consumergroup/_meta/data.json b/metricbeat/module/kafka/consumergroup/_meta/data.json index 2bce75648f32..ca13e283ee47 100644 --- a/metricbeat/module/kafka/consumergroup/_meta/data.json +++ b/metricbeat/module/kafka/consumergroup/_meta/data.json @@ -7,28 +7,22 @@ }, "kafka": { "broker": { - "address": "localhost:32768", + "address": "172.21.0.2:9092", "id": 0 }, "consumergroup": { - "broker": { - "address": "localhost:32768", - "id": 0 - }, "client": { "host": "127.0.0.1", "id": "consumer-1", - "member_id": "consumer-1-a12ac7d4-00aa-45a0-8b35-0a9c6e880bf4" + "member_id": "consumer-1-8653cb3a-afed-4b1b-87d0-2a208319b41e" }, - "consumer_lag": 1059, + "consumer_lag": 77, "error": { "code": 0 }, - "id": "console-consumer-50413", + "id": "console-consumer-40539", "meta": "", - "offset": -1, - "partition": 0, - "topic": "test" + "offset": -1 }, "partition": { "id": 0, @@ -43,7 +37,7 @@ "period": 10000 }, "service": { - "address": "localhost:32768", + "address": "172.21.0.2:9092", "type": "kafka" } } \ No newline at end of file diff --git a/metricbeat/module/kafka/consumergroup/_meta/fields.yml b/metricbeat/module/kafka/consumergroup/_meta/fields.yml index 0f197d3c178d..4939cd45efa0 100644 --- a/metricbeat/module/kafka/consumergroup/_meta/fields.yml +++ b/metricbeat/module/kafka/consumergroup/_meta/fields.yml @@ -4,37 +4,10 @@ consumergroup release: ga fields: - - name: broker - type: group - deprecated: 6.5 - description: > - Broker Consumer Group Information have been read from (Broker handling - the consumer group). - fields: - - name: id - type: long - description: > - Broker id - - - name: address - type: keyword - description: > - Broker address - - name: id type: keyword description: Consumer Group ID - - name: topic - type: keyword - deprecated: 6.5 - description: Topic name - - - name: partition - type: long - deprecated: 6.5 - description: Partition ID - - name: offset type: long description: consumer offset into partition being read diff --git a/metricbeat/module/kafka/consumergroup/consumergroup.go b/metricbeat/module/kafka/consumergroup/consumergroup.go index 25c0205a9f3b..875d0f38cc50 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup.go @@ -94,19 +94,21 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { // Helpful IDs to avoid scripts on queries partitionTopicID := fmt.Sprintf("%d-%s", event["partition"], event["topic"]) - // TODO (deprecation): Remove fields from MetricSetFields moved to ModuleFields - event["broker"] = brokerInfo - r.Event(mb.Event{ - ModuleFields: common.MapStr{ - "broker": brokerInfo, - "topic": common.MapStr{ - "name": event["topic"], - }, - "partition": common.MapStr{ - "id": event["partition"], - "topic_id": partitionTopicID, - }, + moduleFields := common.MapStr{ + "broker": brokerInfo, + "topic": common.MapStr{ + "name": event["topic"], + }, + "partition": common.MapStr{ + "id": event["partition"], + "topic_id": partitionTopicID, }, + } + delete(event, "topic") + delete(event, "partition") + + r.Event(mb.Event{ + ModuleFields: moduleFields, MetricSetFields: event, }) } diff --git a/metricbeat/module/kafka/fields.go b/metricbeat/module/kafka/fields.go index a29309be05f8..4915906dbebb 100644 --- a/metricbeat/module/kafka/fields.go +++ b/metricbeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded zlib format compressed contents of module/kafka. func AssetKafka() string { - return "eJzUWk9v3LYSv/tTDHJyDlFO7x18eMBrUhRumiZIU6DoReCSo13WEqmQ1NqbT1+QlLT6S1Grddr45JU08/txhhwOZ/gKHvB0Bw8keyA3AIabHO/gxTv7+8UNAENNFS8Nl+IO/ncDAODeQSFZleMNgD5IZVIqRcb3d5CRXNunCnMkGu9gb9VmHHOm75z4KxCkwDOk/TOn0n6qZFXWTyZw+2q6qnZKPqBqH0/pm9Xp/35wGuCNFLoqUMFPVhTuRSZVQawAHMgRYYcoQCFhkClZwG0tdiCC5VzseyrNAYE2+hyVl0nng+FYuuPhrPe4GU8uBxDBIXWGxdnNJA5hTKHWk2APeHqUakgkDo+wIyrDNbIWYuQzI0tOE/v/yG9j6ADsZ6vH6ZzDQKWkSqhkY6SBRRdhnCqwqpIxWkmU4VY26flvLdLHRg1wFkRxo0snsML264H9LviXCoEzkJmbseUZXbgH3obLPPwa/DZ0gAjmfnnQ5CoBoZ67BRrFqfYL3Ie6+s3P7//oyLYBboeGRK7rYodEhFbUe/sBmAMxYA5cAx5RGODaohGDDIyMXqwNqMIvFWqT0AMRAvPkS4UVJpp/xRCTzwcE+03jiFoLOOm46DQkUCrJKopJRniOLC1RpRqpFMEYY3koYhwPLwi1nkavhhIVTGryxLJcEhNklqGhh8t50ZxbNzktraGstkrhFdj17bZESlTFDlXAXFewUTyHoGlWMylzTt1unORIGKoUc6T291DViJH/Hprvnes2wFeC5khEupZGLXcNOhq1tlS+SvmAWKJKGNdUCoHULNH4U8p3TgZoLu0uXSvbMFnHdPCp5Goxxpyp+O+fh4tN2aTIT/FsGolnoaNPgq7xkVtDtW+3ccnlPsnySh/SiSk3XjVyD+7rSyZoneChSbhIdieDugmtS7BcUFlwsQcr5aOsHbBTeDEJWZl1LGRl9vLaLBT+hdQgW0elkboalQK1JnvUKQ+mIz1n1DLb4K8zHS4AvYL7L0C9lrtXQm91bwRcA9WccNfl2u05eyLbbt99p/m2y3WiwmvBBS+qwq8oYuDxwOmhXzfQKJjup08ajAQyPuLETAw/DWvti2kcOaKyU+Kczjn5hh2DTCogoEukPOO0PpttyHepVGwLvVrDmeCZyyTXlQTXBq7mfNBYza0ze46VPSevXdzkKc1JsBDkJhd5cpOrnUpjmSWkNmFJqSwKPjo6zA5YZplGm7E4KTveNptZScEVCbfDv+vUGqPtHB9E24PgOaz5YOofuC8viKnDELq6qjmnqF+cbf5CkTQQbYbKPadSIbUR9A7+m/wnZL/ZIuI1a7GwXI+dswCE6rIQqs1GDBV6NdrmySz+dL0WFmq263gMMaYj4XRsj6gZD736Ngw0H6bnsKInXqdwHKTQliDjql6rOJxrvUuG8DEtnkIHpJ30dVzkwshOYXWHdluyCynMoOjnXbDG67TSRhbd+GgIMGIIaKO6i3USuRGb2PZWWiAne5cItKN/7XMWSnJa+YyPaBcpGM8yVCioDTTm0caafj26NiYRbGjg8GAmuxHxQxmvXbdFthxenxl2mxVh+7oCw4rgHuTzf635XiBr6hZ2ZtkZ5nLrOtOfW1MbAnBk7HvjSd2/hVtvOI3GWHqebcLZy+U4fJB6aK5LifRUzQIWaFPcdPvwuTCoBMkHO2EN0I1CofC3OhGZUrI+CQnEwEvm6ZHwnOxyrPXqptWx50cUnf7Wyjkq8BED0+PyROFXp7gJPMMG2ZBmx2w5ex5CH5ziZUIXbKsX+PO8l9qN5VukdqFtPoIwjJq9zdNZlr5v8Ayu/MU3JDiDW5/qj3LjUSPkGVh88pqnacy7TuiToOkSrZ2U+bgcFMnsXjBufa2BZ40BgGvgguYVQ9a0p7l4Zcm0zSK0+wvc3v/2KWokOl3w8LMMwrQNsmWKs+kLXMH/P7YZi08TXN3Lbs6xQSVw4SPIb9N51YyuhsxTG9x3gesdZ8a8Yg849SWKNZeOtp/u+XTxtOZyycWk7Zxmj77jpMj31FdWvT/WUlNV7/bdd1r1Jk02le4qe3JKXdEzeP52t3kMyYEUshIugfCyNhuVatisXaxtE0MPqeZfMSXHxfLoXG1bz52EooAL8rQE3NRlo4ED92qoVCzVKFhUu2G+Xm6xt9btU4VGnS4mYhRHVququx5bCbmI/C8i5PsGdRfnH3bW2mXS2GF8a2wN4IrlsQS4dNssyu9n4zYB/SqXynQphcbLGXj5DRS4TB8JX5xjLeT96w9gBcDwmfRkHmt1Y73fFfM9dlkZV5czHVYredQtlyirl+eNOKrp/XcAAAD//1uFyto=" + return "eJzUWk1z2zYTvvtX7OTkHMLcfXhn3iadjpumyaTpTKcXDgQsRdQgwACgbPnXdwCQFL9EEqLcNjrZFHafB/uFxVJv4AGPd/BAsgdyA2C5FXgHrz64/1/dADA0VPPSciXv4H83AAD+OygUqwTeAJhcaZtSJTO+v4OMCOOeahRIDN7B3qnNOApm7rz4G5CkwBOk+9hj6ZZqVZX1kwncvpquqp1WD6jbx1P6zuoMnx+8BninpKkK1PCTE4V7mSldECcAOTkg7BAlaCQMMq0KuK3FciKZ4HLfU2lzBNro81ReJ50Fw71098NZ73GzH6EGELNb6myLs5tJHMKYRmMmwR7w+Kj0kMg6PMIOqC03yFqIkc+sKjlN3N8jv42hZ2C/Oj1e5zkM1FrphCo2RhpYdBHGqwKnKhmjlURb7mSTnv9ikT43aoCzWRS/u3QCa95+PbDfJf9WIXAGKvMRW57QpX8QbLjMI+TgP0MHiGT+vwCaXKUg1LFboNWcmpDgodTV3/z88Y+ObFvgdmjJyrwudkjkXEZ9dAvA5sSCzbkBPKC0wI1DIxYZWLU6WRtQjd8qNDahOZESRfKtwgoTw59xjsnXHMGtaRxRawEvva46DQmUWrGKYpIRLpClJerUIFVytsY4HppYzyMIQq2n0WugRA2TmgKxTChiZ5llaGl+OS8quHOT19IaymmrNF6BXd9uS6RkVexQz5jrCjZaz2HWNNFMSsGpP40TgYShTlEgdf8PVY0YhfXQrPeu2wBfSSqQyDSWRi13DToGjXFUnpV6QCxRJ4wbqqREapdo/KnUBy8DVCh3StfKNgTrmA4+lVwv1pgTlbD+Zbi4lk1JcVzPppF4ETrmKGmMj3wO1b7dxkWofZKJyuTpRMiNs0btwa++JEDrBg9twmWyO1o0TWldguWSqoLLPTipUGXdhr3Ci0moysaxUJXdq2uz0PgXUossjkojdTUqBRpD9mhSPtuO9JxRy2yDv044XAB6BfdfgHotd0dCb3XvCrgGqrnhxvXa7T17ottuv/tO+23f66wqrwWXvKiKkFHEwmPOad6fGxiUzPTbJwNWARlfcdYERgjDWvtiG0cOqF1InNo5L9+wY5ApDQRMiZRnnNZ3sw39LlWabaFXazgRPHGZ5BpJMLZwNfeDxmo+z9w9VvWcHJvc5CkVZHYQ5IOLPPngakNpLLOE1DYsKVVFwUdXh7MbVllm0HUsXsrtt+1mIin4IeF2+A+dWeNqO68vou1F8FTWQjEND/zKC2rqsIRGTzXPKeoPZ5vPBRPJFUPC4Uj1fbt4Eij4bt2AYYDUWr/2P5dWdQZIO3Tp5/r6eQZF/3yJ2iytjFVFNw4sAUYsAWN1d0A8idyITaR3pAUE2fuC1+7+bajNlAhahZONGF+EGM8y1CgpuoP1EVEO5m61MYlkQwPPb2Zy6rp+K+PZsi8FLYe3J4bdoey8ff1FapLOMENW8Pm/MXwvkTX3MxdZLsJ8D1F3NC3JgfRUqsFcusFCFI74vguk7t/DbTCcQWsdvcA24ez1tLG6NHJlhua6lEhP1VnAAt1Rnm7fPpcWtSRi8AamBuhWodE8e0vBnVISX2xnauAlcXogXJCdwFqvaUa6e35A2ZnjR8aoxEecCY+JBF9B1n1+9YqbwjN8ETCk2TGbYC9D6JNXvExo0pnnSsDF/jy9H3IHS6zXwuTxBYz0Sxhpcga34Wry+ryn6lHqC7D4EjRP0zhfbqU5Spou0dopJcYXypXM7iXjlLiLAM8aA7ibJpdUVAxZ84KLyzeOTDtuRle54fb+ty+rdmLSBQ+/yCZsO2Jfpni2MYAr+P/HthcIB7C/ObtjbyJd2wwNL0giRxifa6mpEUb73Xc6wiDNkZHuKtcepv4Gu3QLssoSAaRQlfRVMsi6I1fp4eR9cVBBLM1Tw58xJYfFu+65QYU51+6tAi7I09pL9mrgmZekVGmWGpRs1ezo/PDDYW8dwqQarT5eTMRqjqxWVY+wthLyVeM/RCgMgeqR3L/srNg0aeww/glADGBEeiwBLv10YJXfT8ZtCvpVfiFgSiUNXs4gyG+gwFX6SPhijLWQ928/gRMAy4tY/0a/JemPOMMLE1VZP3ywHVaRPOr52Sqrl6eDeNUbjL8DAAD//1EPWgk=" } diff --git a/metricbeat/module/kafka/partition/_meta/data.json b/metricbeat/module/kafka/partition/_meta/data.json index f8a2bbf50854..cb01318a6d3a 100644 --- a/metricbeat/module/kafka/partition/_meta/data.json +++ b/metricbeat/module/kafka/partition/_meta/data.json @@ -1,45 +1,40 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "kafka.partition", + "duration": 115000, + "module": "kafka" }, "kafka": { "broker": { - "address": "172.18.0.2:9092", + "address": "172.21.0.2:9092", "id": 0 }, "partition": { - "broker": { - "address": "172.18.0.2:9092", - "id": 0 - }, "id": 0, "offset": { - "newest": 0, + "newest": 1, "oldest": 0 }, "partition": { - "id": 0, "insync_replica": true, "is_leader": true, "leader": 0, "replica": 0 }, - "topic": { - "name": "foo-1538389014-739473801" - }, - "topic_broker_id": "0-foo-1538389014-739473801-0", - "topic_id": "0-foo-1538389014-739473801" + "topic_broker_id": "0-metricbeat-generate-data-0", + "topic_id": "0-metricbeat-generate-data" }, "topic": { - "name": "foo-1538389014-739473801" + "name": "metricbeat-generate-data" } }, "metricset": { - "host": "kafka:9092", - "module": "kafka", "name": "partition", - "rtt": 115 + "period": 10000 + }, + "service": { + "address": "172.21.0.2:9092", + "type": "kafka" } } \ No newline at end of file diff --git a/metricbeat/module/kafka/partition/_meta/fields.yml b/metricbeat/module/kafka/partition/_meta/fields.yml index cf40ad266b93..681b1c6b3ae1 100644 --- a/metricbeat/module/kafka/partition/_meta/fields.yml +++ b/metricbeat/module/kafka/partition/_meta/fields.yml @@ -23,12 +23,6 @@ description: > Partition data. fields: - - name: id - type: long - deprecated: 6.5 - description: > - Partition id. - - name: leader type: long description: > @@ -52,27 +46,3 @@ type: long description: > Error code from fetching partition. - - - name: topic.error.code - type: long - deprecated: 6.5 - description: > - topic error code. - - name: topic.name - type: keyword - deprecated: 6.5 - description: > - Topic name - - - name: broker.id - type: long - deprecated: 6.5 - description: > - Broker id - - name: broker.address - type: keyword - deprecated: 6.5 - description: > - Broker address - - diff --git a/metricbeat/module/kafka/partition/partition.go b/metricbeat/module/kafka/partition/partition.go index 7a1e8aeebf9d..1fd58cdbbd2b 100644 --- a/metricbeat/module/kafka/partition/partition.go +++ b/metricbeat/module/kafka/partition/partition.go @@ -134,7 +134,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } partitionEvent := common.MapStr{ - "id": partition.ID, "leader": partition.Leader, "replica": id, "is_leader": partition.Leader == id, @@ -158,8 +157,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { "topic_id": partitionTopicID, "topic_broker_id": partitionTopicBrokerID, - "topic": evtTopic, - "broker": evtBroker, "partition": partitionEvent, "offset": common.MapStr{ "newest": offNewest, @@ -167,7 +164,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { }, } - // TODO (deprecation): Remove fields from MetricSetFields moved to ModuleFields sent := r.Event(mb.Event{ ModuleFields: common.MapStr{ "broker": evtBroker,