From 7c9b3988fbd7edcfc91ccc7af9cabdfe6b85cc53 Mon Sep 17 00:00:00 2001 From: Jast <745925668@qq.com> Date: Mon, 20 May 2024 12:06:01 +0800 Subject: [PATCH] [feature] add Apache Pulsar monitor (#2013) Co-authored-by: zhangshenghang Co-authored-by: tomsun28 --- home/docs/help/pulsar.md | 76 +++++ .../current/help/pulsar.md | 152 +++++++++ home/sidebars.json | 3 +- .../src/main/resources/define/app-pulsar.yml | 295 ++++++++++++++++++ 4 files changed, 525 insertions(+), 1 deletion(-) create mode 100644 home/docs/help/pulsar.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/pulsar.md create mode 100644 manager/src/main/resources/define/app-pulsar.yml diff --git a/home/docs/help/pulsar.md b/home/docs/help/pulsar.md new file mode 100644 index 00000000000..2cc520d6189 --- /dev/null +++ b/home/docs/help/pulsar.md @@ -0,0 +1,76 @@ +--- +id: pulsar +title: Monitoring Pulsar Monitoring +sidebar_label: Apache Pulsar +keywords: [open-source monitoring system, open-source database monitoring, HbaseMaster monitoring] +--- +> Collecting and monitoring general performance metrics of Pulsar + +**Protocol Used: HTTP** + +## Configuration Parameters + + +| Parameter Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| Target Host | The monitored endpoint's IPV4, IPV6, or domain name. Note⚠️: Do not include the protocol header (e.g., https://, http://). | +| Port | The webServicePort value of Pulsar, default is 8080. | +| Task Name | The name identifying this monitoring task, must be unique. | +| Query Timeout | Set the connection timeout in milliseconds, default is 3000 milliseconds. | +| Monitoring Interval | Interval time for periodic data collection, in seconds, minimum interval is 30 seconds. | +| Binding Tags | Used for categorizing monitoring resources. | +| Description/Remarks | Additional notes and descriptions for this monitoring task. Users can add more information here. | + +### Collected Metrics + +#### Metric Set: Version Information + + +| Metric Name | Unit | Description | +| ------------ | ---- | ------------------- | +| Version Info | NONE | Version Information | + +#### Metric Set: process_start_time_seconds + + +| Metric Name | Unit | Description | +| ------------------ | ---- | ------------------ | +| Process Start Time | NONE | Process Start Time | + +#### Metric Set: process_open_fds + + +| Metric Name | Unit | Description | +| --------------------- | ---- | ------------------------------- | +| Open File Descriptors | NONE | Number of Open File Descriptors | + +#### Metric Set: process_max_fds + + +| Metric Name | Unit | Description | +| -------------------- | ---- | ---------------------------------- | +| Max File Descriptors | NONE | Maximum Number of File Descriptors | + +#### Metric Set: jvm_memory_pool_allocated_bytes + +Number of bytes of memory allocated in a specific memory pool in the Java Virtual Machine (JVM). In Pulsar, this typically refers to the amount of memory allocated for various purposes in the JVM (such as heap memory, non-heap memory, etc.). + +#### Metric Set: jvm_memory_pool_used_bytes + +Unlike allocated_bytes, this metric shows the actual used memory, not just the allocated memory. + +#### Metric Set: jvm_memory_pool_committed_bytes + +Number of bytes of memory committed in a specific memory pool in the JVM. In the JVM, committed memory is the amount of memory guaranteed to be available for the application to use. This portion of memory is typically locked by the operating system to reduce swapping or garbage collection. + +#### Metric Set: jvm_memory_pool_max_bytes + +Maximum number of bytes of memory that can be allocated in a specific memory pool in the JVM. This is the upper limit on memory usage for that memory pool and helps in setting the memory usage cap. + +#### Metric Set: pulsar_broker_publish_latency + +Message publishing latency on the broker side. + +#### Metric Set: pulsar_metadata_store_ops_latency_ms + +Latency of metadata store operations on the broker side. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/pulsar.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/pulsar.md new file mode 100644 index 00000000000..be83c9ff5f7 --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/pulsar.md @@ -0,0 +1,152 @@ +--- +id: pulsar +title: 监控：Pulsar监控 +sidebar_label: A--- +id: pulsar +title: 监控：Pulsar监控 +sidebar_label: Apache Pulsar +keywords: [开源监控系统, 开源数据库监控, HbaseMaster监控] +--- +> 对Pulsar的通用性能指标进行采集监控 + +**使用协议：HTTP** + +## 配置参数 + + +| 参数名称 | 参数帮助描述 | +| ------------ | ------------------------------------------------------------------------- | +| 目标Host | 被监控的对端IPV4，IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 | +| 端口 | Pulsar的webServiceProt值，默认为8080。 | +| 任务名称 | 标识此监控的名称，名称需要保证唯一性。 | +| 查询超时时间 | 设置连接的超时时间，单位ms毫秒，默认3000毫秒。 | +| 监控周期 | 监控周期性采集数据间隔时间，单位秒，可设置的最小间隔为30秒 | +| 绑定标签 | 用于对监控资源进行分类管理 | +| 描述备注 | 更多标识和描述此监控的备注信息，用户可以在这里备注信息 | + +### 采集指标 + +#### 指标集合：版本信息 + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| ------------ | -------- | ------------ | +| Version Info | 无 | 版本信息 | + +#### 指标集合：process_start_time_seconds + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| ------------------ | -------- | ------------ | +| Process Start Time | 无 | 进程启动时间 | + +#### 指标集合：process_open_fds + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| --------------------- | -------- | ---------------- | +| Open File Descriptors | 无 | 打开的文件描述符 | + +#### 指标集合：process_max_fds + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| -------------------- | -------- | -------------- | +| Max File Descriptors | 无 | 最大文件描述符 | + +#### 指标集合： jvm_memory_pool_allocated_bytes + +Java虚拟机（JVM）中特定内存池已分配的内存字节数。在Pulsar中，这通常指的是用于各种目的的JVM内存（如堆内存、非堆内存等）中已经分配出去的内存量。 + +#### 指标集合：jvm_memory_pool_used_bytes + +与allocated_bytes不同，这个指标会显示实际使用的内存，而不仅仅是分配的内存。 + +#### 指标集合：jvm_memory_pool_committed_bytes + +JVM中特定内存池已承诺的内存字节数。在JVM中，承诺的内存是指虚拟机保证可供应用程序使用的内存量，通常这部分内存会被操作系统锁定，以减少交换或垃圾回收的可能性。 + +#### 指标集合：jvm_memory_pool_max_bytes + +JVM中特定内存池可分配的最大内存字节数。这是该内存池允许的最大内存限制，有助于设置内存使用的上限。 + +#### 指标集合：pulsar_broker_publish_latency + +Broker端消息发布延迟 + +#### 指标集合：pulsar_metadata_store_ops_latency_ms + +Broker端元数据存储操作延迟 +pache Pulsar +keywords: [开源监控系统, 开源数据库监控, HbaseMaster监控] +--- +> 对Pulsar的通用性能指标进行采集监控 + +**使用协议：HTTP** + +## 配置参数 + + +| 参数名称 | 参数帮助描述 | +| ------------ | ------------------------------------------------------------------------- | +| 目标Host | 被监控的对端IPV4，IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 | +| 端口 | Pulsar的webServiceProt值，默认为8080。 | +| 任务名称 | 标识此监控的名称，名称需要保证唯一性。 | +| 查询超时时间 | 设置连接的超时时间，单位ms毫秒，默认3000毫秒。 | +| 监控周期 | 监控周期性采集数据间隔时间，单位秒，可设置的最小间隔为30秒 | +| 绑定标签 | 用于对监控资源进行分类管理 | +| 描述备注 | 更多标识和描述此监控的备注信息，用户可以在这里备注信息 | + +### 采集指标 + +#### 指标集合：版本信息 + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| ------------ | -------- | ------------ | +| Version Info | 无 | 版本信息 | + +#### 指标集合：process_start_time_seconds + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| ------------------ | -------- | ------------ | +| Process Start Time | 无 | 进程启动时间 | + +#### 指标集合：process_open_fds + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| --------------------- | -------- | ---------------- | +| Open File Descriptors | 无 | 打开的文件描述符 | + +#### 指标集合：process_max_fds + + +| 指标名称 | 指标单位 | 指标帮助描述 | +| -------------------- | -------- | -------------- | +| Max File Descriptors | 无 | 最大文件描述符 | + +#### 指标集合： jvm_memory_pool_allocated_bytes + +Java虚拟机（JVM）中特定内存池已分配的内存字节数。在Pulsar中，这通常指的是用于各种目的的JVM内存（如堆内存、非堆内存等）中已经分配出去的内存量。 + +#### 指标集合：jvm_memory_pool_used_bytes + +与allocated_bytes不同，这个指标会显示实际使用的内存，而不仅仅是分配的内存。 + +#### 指标集合：jvm_memory_pool_committed_bytes + +JVM中特定内存池已承诺的内存字节数。在JVM中，承诺的内存是指虚拟机保证可供应用程序使用的内存量，通常这部分内存会被操作系统锁定，以减少交换或垃圾回收的可能性。 + +#### 指标集合：jvm_memory_pool_max_bytes + +JVM中特定内存池可分配的最大内存字节数。这是该内存池允许的最大内存限制，有助于设置内存使用的上限。 + +#### 指标集合：pulsar_broker_publish_latency + +Broker端消息发布延迟 + +#### 指标集合：pulsar_metadata_store_ops_latency_ms + +Broker端元数据存储操作延迟 diff --git a/home/sidebars.json b/home/sidebars.json index 02966b270da..10d0c4e8465 100755 --- a/home/sidebars.json +++ b/home/sidebars.json @@ -220,7 +220,8 @@ "help/rocketmq", "help/spring_gateway", "help/nacos", - "help/kafka" + "help/kafka", + "help/pulsar" ] }, { diff --git a/manager/src/main/resources/define/app-pulsar.yml b/manager/src/main/resources/define/app-pulsar.yml new file mode 100644 index 00000000000..80f8ae945bb --- /dev/null +++ b/manager/src/main/resources/define/app-pulsar.yml @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The monitoring type category: service-application service monitoring, db-database monitoring, custom-custom monitoring. +category: mid +# The monitoring application type name (consistent with the file name) +app: pulsar +name: + zh-CN: Pulsar + en-US: Pulsar +# The description and help of this monitoring type +help: + zh-CN: Hertzbeat 从 Pulsar WebService接口查询到 Pulsar 的通用指标数据来进行监控。
您可以点击 “新建 Pulsar” 并进行配置，或者选择“更多操作”，导入已有配置。 + en-US: Hertzbeat queries general metric data from Pulsar through the Pulsar WebService interface for monitoring. You can click "Create Pulsar" to configure a new monitoring task, or select "More Actions" to import existing configurations. + zh-TW: Hertzbeat 從 Pulsar WebService接口查詢到 Pulsar 的通用指標數據來進行監控。
您可以點擊 “新建 Pulsar” 並進行配置，或者選擇“更多操作”，導入已有配置。 +helpLink: + zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/pulsar + en-US: https://hertzbeat.apache.org/docs/help/pulsar +# Input params define for monitoring(render web ui by the definition) +params: + # field-param field key + - field: host + # name-param field display i18n name + name: + zh-CN: 目标Host + en-US: Target Host + # type-param field type(most mapping the html input type) + type: host + # required-true or false + required: true + - field: port + name: + zh-CN: 端口 + en-US: Port + # type-param field type(most mapping the html input type) + type: number + # when type is number, range is required + range: '[0,65535]' + # default value + defaultValue: 8030 + required: true + - field: timeout + name: + zh-CN: 查询超时时间 + en-US: Query Timeout + type: number + required: false + # hide param-true or false + hide: true + defaultValue: 6000 +metrics: + - name: pulsar_version_info + i18n: + zh-CN: 版本信息 + en-US: Version Info + priority: 0 + fields: + - field: version + i18n: + zh-CN: 版本信息 + en-US: Version Info + type: 1 + label: false + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: process_start_time_seconds + priority: 0 + fields: + - field: value + i18n: + zh-CN: 进程启动时间 + en-US: Process Start Time + type: 0 + label: false + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: process_open_fds + priority: 0 + fields: + - field: value + type: 0 + i18n: + zh-CN: 打开的文件描述符 + en-US: Open File Descriptors + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: process_max_fds + priority: 0 + fields: + - field: value + type: 0 + i18n: + zh-CN: 最大文件描述符 + en-US: Max File Descriptors + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: jvm_memory_pool_allocated_bytes + priority: 0 + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + fields: + - field: .name + type: 1 + i18n: + zh-CN: 指标名称 + en-US: Metric Name + - field: pool + type: 1 + - field: value + type: 0 + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: jvm_memory_pool_bytes_used + priority: 0 + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + fields: + - field: pool + type: 1 + - field: value + type: 0 + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: jvm_memory_pool_bytes_committed + priority: 0 + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + fields: + - field: pool + type: 1 + - field: value + type: 0 + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: jvm_memory_pool_bytes_max + priority: 0 + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + fields: + - field: pool + type: 1 + - field: value + type: 0 + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: pulsar_broker_publish_latency + priority: 0 + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + fields: + - field: .name + type: 1 + i18n: + zh-CN: 指标名称 + en-US: Metric Name + - field: cluster + type: 1 + - field: quantile + type: 1 + - field: value + type: 0 + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus + - name: pulsar_metadata_store_ops_latency_ms + priority: 0 + # the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk + protocol: http + fields: + - field: .name + type: 1 + i18n: + zh-CN: 指标名称 + en-US: Metric Name + - field: cluster + type: 1 + - field: name + type: 1 + - field: type + type: 1 + - field: status + type: 1 + - field: le + type: 0 + # the config content when protocol is http + http: + # http host: ipv4 ipv6 domain + host: ^_^host^_^ + # port + port: ^_^port^_^ + # url request interface path + url: '/metrics' + timeout: ^_^timeout^_^ + method: GET + parseType: prometheus \ No newline at end of file