[Metricbeat] Update Ceph module to support new API (elastic#16254)

* add new ceph metricsets * adjust system tests (cherry picked from commit 273c6a8)
mtojek · Feb 19, 2020 · abfcc97 · abfcc97
1 parent 8da2bfe
commit abfcc97
Show file tree

Hide file tree

Showing 99 changed files with 2,636 additions and 14 deletions.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -3706,6 +3706,116 @@ type: long
 
 --
 
+[float]
+=== mgr_cluster_disk
+
+see: cluster_disk
+
+
+[float]
+=== mgr_cluster_health
+
+see: cluster_health
+
+
+[float]
+=== mgr_osd_perf
+
+OSD performance metrics of Ceph cluster
+
+
+
+*`ceph.mgr_osd_perf.id`*::
++
+--
+OSD ID
+
+type: long
+
+--
+
+*`ceph.mgr_osd_perf.stats.commit_latency_ms`*::
++
+--
+Commit latency in ms
+
+type: long
+
+--
+
+*`ceph.mgr_osd_perf.stats.apply_latency_ms`*::
++
+--
+Apply latency in ms
+
+type: long
+
+--
+
+*`ceph.mgr_osd_perf.stats.commit_latency_ns`*::
++
+--
+Commit latency in ns
+
+type: long
+
+--
+
+*`ceph.mgr_osd_perf.stats.apply_latency_ns`*::
++
+--
+Apply latency in ns
+
+type: long
+
+--
+
+[float]
+=== mgr_osd_pool_stats
+
+OSD pool stats of Ceph cluster
+
+
+
+*`ceph.mgr_osd_pool_stats.pool_name`*::
++
+--
+Pool name
+
+type: keyword
+
+--
+
+*`ceph.mgr_osd_pool_stats.pool_id`*::
++
+--
+Pool ID
+
+type: long
+
+--
+
+*`ceph.mgr_osd_pool_stats.client_io_rate`*::
++
+--
+Client I/O rates
+
+type: object
+
+--
+
+[float]
+=== mgr_osd_tree
+
+see: osd_tree
+
+
+[float]
+=== mgr_pool_disk
+
+see: pool_disk
+
+
 [float]
 === monitor_health
 

diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc
@@ -11,7 +11,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de
 [float]
 === Compatibility
 
-The Ceph module is tested with Ceph Jewel (10.2.10).
+The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7).
+
+Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon.
 
 [float]
 === Dashboard
@@ -52,6 +54,18 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-ceph-cluster_status,cluster_status>>
 
+* <<metricbeat-metricset-ceph-mgr_cluster_disk,mgr_cluster_disk>>
+
+* <<metricbeat-metricset-ceph-mgr_cluster_health,mgr_cluster_health>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_perf,mgr_osd_perf>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_pool_stats,mgr_osd_pool_stats>>
+
+* <<metricbeat-metricset-ceph-mgr_osd_tree,mgr_osd_tree>>
+
+* <<metricbeat-metricset-ceph-mgr_pool_disk,mgr_pool_disk>>
+
 * <<metricbeat-metricset-ceph-monitor_health,monitor_health>>
 
 * <<metricbeat-metricset-ceph-osd_df,osd_df>>
@@ -66,6 +80,18 @@ include::ceph/cluster_health.asciidoc[]
 
 include::ceph/cluster_status.asciidoc[]
 
+include::ceph/mgr_cluster_disk.asciidoc[]
+
+include::ceph/mgr_cluster_health.asciidoc[]
+
+include::ceph/mgr_osd_perf.asciidoc[]
+
+include::ceph/mgr_osd_pool_stats.asciidoc[]
+
+include::ceph/mgr_osd_tree.asciidoc[]
+
+include::ceph/mgr_pool_disk.asciidoc[]
+
 include::ceph/monitor_health.asciidoc[]
 
 include::ceph/osd_df.asciidoc[]

diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_cluster_disk]]
+=== Ceph mgr_cluster_disk metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_cluster_disk/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/ceph/mgr_cluster_disk/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_cluster_health]]
+=== Ceph mgr_cluster_health metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_cluster_health/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/ceph/mgr_cluster_health/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_perf]]
+=== Ceph mgr_osd_perf metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_perf/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/ceph/mgr_osd_perf/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_pool_stats]]
+=== Ceph mgr_osd_pool_stats metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/ceph/mgr_osd_pool_stats/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_osd_tree]]
+=== Ceph mgr_osd_tree metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_osd_tree/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/ceph/mgr_osd_tree/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-ceph-mgr_pool_disk]]
+=== Ceph mgr_pool_disk metricset
+
+beta[]
+
+include::../../../module/ceph/mgr_pool_disk/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-ceph,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/ceph/mgr_pool_disk/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -38,9 +38,15 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-beat-state,state>>   
 |<<metricbeat-metricset-beat-stats,stats>>   
 |<<metricbeat-module-ceph,Ceph>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.7+| .7+|  |<<metricbeat-metricset-ceph-cluster_disk,cluster_disk>>   
+.13+| .13+|  |<<metricbeat-metricset-ceph-cluster_disk,cluster_disk>>   
 |<<metricbeat-metricset-ceph-cluster_health,cluster_health>>   
 |<<metricbeat-metricset-ceph-cluster_status,cluster_status>>   
+|<<metricbeat-metricset-ceph-mgr_cluster_disk,mgr_cluster_disk>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_cluster_health,mgr_cluster_health>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_perf,mgr_osd_perf>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_pool_stats,mgr_osd_pool_stats>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_osd_tree,mgr_osd_tree>> beta[]  
+|<<metricbeat-metricset-ceph-mgr_pool_disk,mgr_pool_disk>> beta[]  
 |<<metricbeat-metricset-ceph-monitor_health,monitor_health>>   
 |<<metricbeat-metricset-ceph-osd_df,osd_df>>   
 |<<metricbeat-metricset-ceph-osd_tree,osd_tree>>   

diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go
diff --git a/metricbeat/module/ceph/_meta/Dockerfile → ...icbeat/module/ceph/_meta/Dockerfile.jewel b/metricbeat/module/ceph/_meta/Dockerfile → ...icbeat/module/ceph/_meta/Dockerfile.jewel
@@ -8,7 +8,6 @@ HEALTHCHECK --interval=1s --retries=300 \
   CMD curl -s -H "Accept: application/json" localhost:5000/api/v0.1/health \
         | jq .output.health.health_services[0].mons[0] \
         | grep health
-EXPOSE 5000
 
 ENV NETWORK_AUTO_DETECT 4
 ENV DEMO_DAEMONS osd,rest_api

diff --git a/metricbeat/module/ceph/_meta/Dockerfile.nautilus b/metricbeat/module/ceph/_meta/Dockerfile.nautilus
@@ -0,0 +1,19 @@
+ARG CEPH_VERSION
+FROM ceph/daemon:${CEPH_VERSION}
+
+RUN yum -q install -y jq && yum clean all && rm -fr /var/cache/yum
+
+# Wait for the health endpoint to have monitors information
+ADD healthcheck.sh /
+HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh
+
+EXPOSE 5000 8003 9283
+
+ENV NETWORK_AUTO_DETECT 4
+ENV CEPH_DAEMON demo
+ENV CEPH_DEMO_UID beats
+ENV CEPH_DEMO_BUCKET beats
+ENV CEPH_DEMO_ACCESS_KEY demo
+ENV CEPH_DEMO_SECRET_KEY demo
+
+CMD ["demo"]
diff --git a/metricbeat/module/ceph/_meta/config-mgr.yml b/metricbeat/module/ceph/_meta/config-mgr.yml
@@ -0,0 +1,20 @@
+- module: ceph
+  metricsets:
+    - mgr_cluster_health
+  period: 10s
+  hosts: [ "https://localhost:8003" ]
+  #username: "user"
+  #password: "secret"
+
+- module: ceph
+  metricsets:
+    - mgr_cluster_disk
+    - mgr_cluster_health
+    - mgr_osd_disk
+    - mgr_osd_perf
+  #  - mgr_osd_pool_stats
+  #  - mgr_osd_tree
+  period: 1m
+  hosts: [ "https://localhost:8003" ]
+  #username: "user"
+  #password: "secret"
diff --git a/metricbeat/module/ceph/_meta/docs.asciidoc b/metricbeat/module/ceph/_meta/docs.asciidoc
@@ -4,7 +4,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de
 [float]
 === Compatibility
 
-The Ceph module is tested with Ceph Jewel (10.2.10).
+The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7).
+
+Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon.
 
 [float]
 === Dashboard