From 60b76658980a68f6dc7793308c5966e23e5d09dc Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 19 Feb 2020 12:02:24 +0100 Subject: [PATCH] [Metricbeat] Update Ceph module to support new API (#16254) * add new ceph metricsets * adjust system tests --- metricbeat/docs/fields.asciidoc | 110 +++++++++++++++++ metricbeat/docs/modules/ceph.asciidoc | 28 ++++- .../modules/ceph/mgr_cluster_disk.asciidoc | 23 ++++ .../modules/ceph/mgr_cluster_health.asciidoc | 23 ++++ .../docs/modules/ceph/mgr_osd_perf.asciidoc | 23 ++++ .../modules/ceph/mgr_osd_pool_stats.asciidoc | 23 ++++ .../docs/modules/ceph/mgr_osd_tree.asciidoc | 23 ++++ .../docs/modules/ceph/mgr_pool_disk.asciidoc | 23 ++++ metricbeat/docs/modules_list.asciidoc | 8 +- metricbeat/include/list_common.go | 6 + .../_meta/{Dockerfile => Dockerfile.jewel} | 1 - .../module/ceph/_meta/Dockerfile.nautilus | 19 +++ metricbeat/module/ceph/_meta/config-mgr.yml | 20 +++ metricbeat/module/ceph/_meta/docs.asciidoc | 4 +- metricbeat/module/ceph/_meta/healthcheck.sh | 14 +++ .../module/ceph/_meta/supported-versions.yml | 5 + .../cluster_disk_integration_test.go | 2 +- .../cluster_health_integration_test.go | 2 +- .../cluster_status_integration_test.go | 2 +- metricbeat/module/ceph/docker-compose.yml | 16 ++- metricbeat/module/ceph/fields.go | 2 +- metricbeat/module/ceph/mgr/event_mapping.go | 65 ++++++++++ metricbeat/module/ceph/mgr/metricset.go | 52 ++++++++ .../ceph/mgr_cluster_disk/_meta/data.json | 32 +++++ .../ceph/mgr_cluster_disk/_meta/docs.asciidoc | 1 + .../ceph/mgr_cluster_disk/_meta/fields.yml | 6 + .../_meta/testdata/config.yml | 3 + .../_meta/testdata/failed.json | 17 +++ .../_meta/testdata/failed.json-expected.json | 3 + .../_meta/testdata/sample.json | 17 +++ .../_meta/testdata/sample.json-expected.json | 3 + .../module/ceph/mgr_cluster_disk/data.go | 53 ++++++++ .../ceph/mgr_cluster_disk/mgr_cluster_disk.go | 81 ++++++++++++ .../mgr_cluster_disk_integration_test.go | 52 ++++++++ .../mgr_cluster_disk/mgr_cluster_disk_test.go | 29 +++++ .../ceph/mgr_cluster_health/_meta/data.json | 31 +++++ .../mgr_cluster_health/_meta/docs.asciidoc | 1 + .../ceph/mgr_cluster_health/_meta/fields.yml | 6 + .../_meta/testdata/failed.json | 17 +++ .../_meta/testdata/status.json | 17 +++ .../_meta/testdata/time_sync_status.json | 17 +++ .../module/ceph/mgr_cluster_health/data.go | 64 ++++++++++ .../mgr_cluster_health/mgr_cluster_health.go | 90 ++++++++++++++ .../mgr_cluster_health_integration_test.go | 51 ++++++++ .../mgr_cluster_health_test.go | 116 ++++++++++++++++++ .../module/ceph/mgr_osd_perf/_meta/data.json | 26 ++++ .../ceph/mgr_osd_perf/_meta/docs.asciidoc | 1 + .../module/ceph/mgr_osd_perf/_meta/fields.yml | 21 ++++ .../mgr_osd_perf/_meta/testdata/config.yml | 3 + .../mgr_osd_perf/_meta/testdata/failed.json | 17 +++ .../_meta/testdata/failed.json-expected.json | 3 + .../mgr_osd_perf/_meta/testdata/sample.json | 17 +++ .../_meta/testdata/sample.json-expected.json | 3 + metricbeat/module/ceph/mgr_osd_perf/data.go | 62 ++++++++++ .../module/ceph/mgr_osd_perf/mgr_osd_perf.go | 82 +++++++++++++ .../mgr_osd_perf_integration_test.go | 52 ++++++++ .../ceph/mgr_osd_perf/mgr_osd_perf_test.go | 29 +++++ .../ceph/mgr_osd_pool_stats/_meta/data.json | 27 ++++ .../mgr_osd_pool_stats/_meta/docs.asciidoc | 1 + .../ceph/mgr_osd_pool_stats/_meta/fields.yml | 28 +++++ .../_meta/testdata/config.yml | 3 + .../_meta/testdata/failed.json | 17 +++ .../_meta/testdata/failed.json-expected.json | 3 + .../_meta/testdata/sample.json | 17 +++ .../_meta/testdata/sample.json-expected.json | 10 ++ .../module/ceph/mgr_osd_pool_stats/data.go | 60 +++++++++ .../mgr_osd_pool_stats/mgr_osd_pool_stats.go | 82 +++++++++++++ .../mgr_osd_pool_stats_integration_test.go | 52 ++++++++ .../mgr_osd_pool_stats_test.go | 29 +++++ .../module/ceph/mgr_osd_tree/_meta/data.json | 27 ++++ .../ceph/mgr_osd_tree/_meta/docs.asciidoc | 1 + .../module/ceph/mgr_osd_tree/_meta/fields.yml | 6 + .../mgr_osd_tree/_meta/testdata/config.yml | 3 + .../mgr_osd_tree/_meta/testdata/failed.json | 17 +++ .../_meta/testdata/failed.json-expected.json | 3 + .../mgr_osd_tree/_meta/testdata/sample.json | 17 +++ .../_meta/testdata/sample.json-expected.json | 6 + metricbeat/module/ceph/mgr_osd_tree/data.go | 106 ++++++++++++++++ .../module/ceph/mgr_osd_tree/mgr_osd_tree.go | 86 +++++++++++++ .../mgr_osd_tree_integration_test.go | 52 ++++++++ .../ceph/mgr_osd_tree/mgr_osd_tree_test.go | 29 +++++ .../module/ceph/mgr_pool_disk/_meta/data.json | 31 +++++ .../ceph/mgr_pool_disk/_meta/docs.asciidoc | 1 + .../ceph/mgr_pool_disk/_meta/fields.yml | 6 + .../mgr_pool_disk/_meta/testdata/config.yml | 3 + .../mgr_pool_disk/_meta/testdata/failed.json | 17 +++ .../_meta/testdata/failed.json-expected.json | 3 + .../mgr_pool_disk/_meta/testdata/sample.json | 17 +++ .../_meta/testdata/sample.json-expected.json | 10 ++ metricbeat/module/ceph/mgr_pool_disk/data.go | 67 ++++++++++ .../ceph/mgr_pool_disk/mgr_pool_disk.go | 86 +++++++++++++ .../mgr_pool_disk_integration_test.go | 52 ++++++++ .../ceph/mgr_pool_disk/mgr_pool_disk_test.go | 29 +++++ metricbeat/module/ceph/mgrtest/password.go | 39 ++++++ .../ceph/osd_df/osd_df_integration_test.go | 2 +- .../osd_tree/osd_tree_integration_test.go | 2 +- .../pool_disk/pool_disk_integration_test.go | 2 +- metricbeat/module/ceph/test_ceph.py | 61 ++++++++- metricbeat/modules.d/ceph-mgr.yml.disabled | 23 ++++ 99 files changed, 2636 insertions(+), 14 deletions(-) create mode 100644 metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc rename metricbeat/module/ceph/_meta/{Dockerfile => Dockerfile.jewel} (97%) create mode 100644 metricbeat/module/ceph/_meta/Dockerfile.nautilus create mode 100644 metricbeat/module/ceph/_meta/config-mgr.yml create mode 100755 metricbeat/module/ceph/_meta/healthcheck.sh create mode 100644 metricbeat/module/ceph/_meta/supported-versions.yml create mode 100644 metricbeat/module/ceph/mgr/event_mapping.go create mode 100644 metricbeat/module/ceph/mgr/metricset.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/data.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/failed.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/status.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/time_sync_status.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/data.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/data.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/data.go create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/data.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/data.go create mode 100644 metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go create mode 100644 metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go create mode 100644 metricbeat/module/ceph/mgrtest/password.go create mode 100644 metricbeat/modules.d/ceph-mgr.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 754041ae8ba..6394e568e15 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3932,6 +3932,116 @@ type: long -- +[float] +=== mgr_cluster_disk + +see: cluster_disk + + +[float] +=== mgr_cluster_health + +see: cluster_health + + +[float] +=== mgr_osd_perf + +OSD performance metrics of Ceph cluster + + + +*`ceph.mgr_osd_perf.id`*:: ++ +-- +OSD ID + +type: long + +-- + +*`ceph.mgr_osd_perf.stats.commit_latency_ms`*:: ++ +-- +Commit latency in ms + +type: long + +-- + +*`ceph.mgr_osd_perf.stats.apply_latency_ms`*:: ++ +-- +Apply latency in ms + +type: long + +-- + +*`ceph.mgr_osd_perf.stats.commit_latency_ns`*:: ++ +-- +Commit latency in ns + +type: long + +-- + +*`ceph.mgr_osd_perf.stats.apply_latency_ns`*:: ++ +-- +Apply latency in ns + +type: long + +-- + +[float] +=== mgr_osd_pool_stats + +OSD pool stats of Ceph cluster + + + +*`ceph.mgr_osd_pool_stats.pool_name`*:: ++ +-- +Pool name + +type: keyword + +-- + +*`ceph.mgr_osd_pool_stats.pool_id`*:: ++ +-- +Pool ID + +type: long + +-- + +*`ceph.mgr_osd_pool_stats.client_io_rate`*:: ++ +-- +Client I/O rates + +type: object + +-- + +[float] +=== mgr_osd_tree + +see: osd_tree + + +[float] +=== mgr_pool_disk + +see: pool_disk + + [float] === monitor_health diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index 1fb296f1784..ffbe40413e2 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -11,7 +11,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de [float] === Compatibility -The Ceph module is tested with Ceph Jewel (10.2.10). +The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7). + +Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon. [float] === Dashboard @@ -52,6 +54,18 @@ The following metricsets are available: * <> +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + * <> * <> @@ -66,6 +80,18 @@ include::ceph/cluster_health.asciidoc[] include::ceph/cluster_status.asciidoc[] +include::ceph/mgr_cluster_disk.asciidoc[] + +include::ceph/mgr_cluster_health.asciidoc[] + +include::ceph/mgr_osd_perf.asciidoc[] + +include::ceph/mgr_osd_pool_stats.asciidoc[] + +include::ceph/mgr_osd_tree.asciidoc[] + +include::ceph/mgr_pool_disk.asciidoc[] + include::ceph/monitor_health.asciidoc[] include::ceph/osd_df.asciidoc[] diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc new file mode 100644 index 00000000000..6241a39a8f3 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_cluster_disk]] +=== Ceph mgr_cluster_disk metricset + +beta[] + +include::../../../module/ceph/mgr_cluster_disk/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_cluster_disk/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc new file mode 100644 index 00000000000..3e2ce6b273b --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_cluster_health]] +=== Ceph mgr_cluster_health metricset + +beta[] + +include::../../../module/ceph/mgr_cluster_health/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_cluster_health/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc new file mode 100644 index 00000000000..d3549734d2f --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_perf]] +=== Ceph mgr_osd_perf metricset + +beta[] + +include::../../../module/ceph/mgr_osd_perf/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_perf/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc new file mode 100644 index 00000000000..75c986659ef --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_pool_stats]] +=== Ceph mgr_osd_pool_stats metricset + +beta[] + +include::../../../module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_pool_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc new file mode 100644 index 00000000000..3153fd50726 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_tree]] +=== Ceph mgr_osd_tree metricset + +beta[] + +include::../../../module/ceph/mgr_osd_tree/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_tree/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc new file mode 100644 index 00000000000..4e64a31f124 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_pool_disk]] +=== Ceph mgr_pool_disk metricset + +beta[] + +include::../../../module/ceph/mgr_pool_disk/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_pool_disk/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index e35f54b7351..bf18f416ef0 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -38,9 +38,15 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.7+| .7+| |<> +.13+| .13+| |<> |<> |<> +|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> |<> |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 0903128c7bf..55291b93af1 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -32,6 +32,12 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/ceph/cluster_disk" _ "github.com/elastic/beats/v7/metricbeat/module/ceph/cluster_health" _ "github.com/elastic/beats/v7/metricbeat/module/ceph/cluster_status" + _ "github.com/elastic/beats/v7/metricbeat/module/ceph/mgr_cluster_disk" + _ "github.com/elastic/beats/v7/metricbeat/module/ceph/mgr_cluster_health" + _ "github.com/elastic/beats/v7/metricbeat/module/ceph/mgr_osd_perf" + _ "github.com/elastic/beats/v7/metricbeat/module/ceph/mgr_osd_pool_stats" + _ "github.com/elastic/beats/v7/metricbeat/module/ceph/mgr_osd_tree" + _ "github.com/elastic/beats/v7/metricbeat/module/ceph/mgr_pool_disk" _ "github.com/elastic/beats/v7/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/v7/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/v7/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/_meta/Dockerfile b/metricbeat/module/ceph/_meta/Dockerfile.jewel similarity index 97% rename from metricbeat/module/ceph/_meta/Dockerfile rename to metricbeat/module/ceph/_meta/Dockerfile.jewel index 5919c2da625..19d23bd275c 100644 --- a/metricbeat/module/ceph/_meta/Dockerfile +++ b/metricbeat/module/ceph/_meta/Dockerfile.jewel @@ -8,7 +8,6 @@ HEALTHCHECK --interval=1s --retries=300 \ CMD curl -s -H "Accept: application/json" localhost:5000/api/v0.1/health \ | jq .output.health.health_services[0].mons[0] \ | grep health -EXPOSE 5000 ENV NETWORK_AUTO_DETECT 4 ENV DEMO_DAEMONS osd,rest_api diff --git a/metricbeat/module/ceph/_meta/Dockerfile.nautilus b/metricbeat/module/ceph/_meta/Dockerfile.nautilus new file mode 100644 index 00000000000..130d771635a --- /dev/null +++ b/metricbeat/module/ceph/_meta/Dockerfile.nautilus @@ -0,0 +1,19 @@ +ARG CEPH_VERSION +FROM ceph/daemon:${CEPH_VERSION} + +RUN yum -q install -y jq && yum clean all && rm -fr /var/cache/yum + +# Wait for the health endpoint to have monitors information +ADD healthcheck.sh / +HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh + +EXPOSE 5000 8003 9283 + +ENV NETWORK_AUTO_DETECT 4 +ENV CEPH_DAEMON demo +ENV CEPH_DEMO_UID beats +ENV CEPH_DEMO_BUCKET beats +ENV CEPH_DEMO_ACCESS_KEY demo +ENV CEPH_DEMO_SECRET_KEY demo + +CMD ["demo"] diff --git a/metricbeat/module/ceph/_meta/config-mgr.yml b/metricbeat/module/ceph/_meta/config-mgr.yml new file mode 100644 index 00000000000..a4bd4ed78d5 --- /dev/null +++ b/metricbeat/module/ceph/_meta/config-mgr.yml @@ -0,0 +1,20 @@ +- module: ceph + metricsets: + - mgr_cluster_health + period: 10s + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" + +- module: ceph + metricsets: + - mgr_cluster_disk + - mgr_cluster_health + - mgr_osd_disk + - mgr_osd_perf + # - mgr_osd_pool_stats + # - mgr_osd_tree + period: 1m + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" diff --git a/metricbeat/module/ceph/_meta/docs.asciidoc b/metricbeat/module/ceph/_meta/docs.asciidoc index d055dc46dcc..0831de00302 100644 --- a/metricbeat/module/ceph/_meta/docs.asciidoc +++ b/metricbeat/module/ceph/_meta/docs.asciidoc @@ -4,7 +4,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de [float] === Compatibility -The Ceph module is tested with Ceph Jewel (10.2.10). +The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7). + +Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon. [float] === Dashboard diff --git a/metricbeat/module/ceph/_meta/healthcheck.sh b/metricbeat/module/ceph/_meta/healthcheck.sh new file mode 100755 index 00000000000..09f34f38ea7 --- /dev/null +++ b/metricbeat/module/ceph/_meta/healthcheck.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +KEYS_JSON=/opt/ceph-container/sree/static/restful-list-keys.json + +if [[ ! -f "${KEYS_JSON}" ]]; then + ceph restful list-keys | grep demo >/dev/null + if [[ $? -eq 0 ]]; then + ceph restful list-keys > ${KEYS_JSON} + else + exit 1 + fi +fi + +ceph health | grep HEALTH_OK && curl -s localhost:5000 >/dev/null diff --git a/metricbeat/module/ceph/_meta/supported-versions.yml b/metricbeat/module/ceph/_meta/supported-versions.yml new file mode 100644 index 00000000000..b8fc1d529c5 --- /dev/null +++ b/metricbeat/module/ceph/_meta/supported-versions.yml @@ -0,0 +1,5 @@ +variants: + - CEPH_CODENAME: jewel + CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 + - CEPH_CODENAME: nautilus + CEPH_VERSION: master-97985eb-nautilus-centos-7-x86_64 diff --git a/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go b/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go index 90b44ad3799..9c467a459bb 100644 --- a/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go b/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go index 159c591cac7..3a4e3799425 100644 --- a/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go +++ b/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go b/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go index cef6e52fdc5..fee52f71fe1 100644 --- a/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go +++ b/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index e3da5ece1cc..e381bed4080 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -2,10 +2,22 @@ version: '2.3' services: ceph: - image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64}-1 + image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-2 build: context: ./_meta + dockerfile: Dockerfile.${CEPH_CODENAME:-nautilus} args: - CEPH_VERSION: ${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64} + CEPH_VERSION: ${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64} + ports: + - 5000 + - 8003 + - 8080 + ceph-api: + image: docker.elastic.co/integrations-ci/beats-ceph:master-6373c6a-jewel-centos-7-x86_64-1 + build: + context: ./_meta + dockerfile: Dockerfile.jewel + args: + CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 ports: - 5000 diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index ef7a1c6db62..f401f784976 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzEms9v27gSx+/5KwY5vQckxntXHx7Q1y3QYDdtsGlPi4VKkyOJa0okOFS8/u8XpGTZ1k/boV2fWtv5zsczo+HMSI+wxu0SOJr8DsBJp3AJ9x/R5Pd3AAKJW2mc1OUS/ncHAOA/gkKLSuEdAOXauoTrMpXZElKmyL9rUSEjXELG/HfQOVlmtIQ/7onU/QPc586Z+z/vAFKJStAyKD9CyQpsWfzLbY1XsboyzTsDRP71w//RD+C6dEyWBC5HKNBZyf2/mYMNWgTilhkUkFpdwMdPL58XjcAhxhGKqsihTYSkdfvhENYEmn+N6Bz7affqwhwCsTcmFVspXKy2DunoOzsupcus88EEmn992KlCUAWdBgc21J2vp9oWzC2hD7CDdNoxFRXwm1eMA1cRiqhs3wnF5WjdTMuRKZfHyLWe0vnZpt/QMqUScsxVw/5a43ajrTjPZV9rXah1p9zWppQskOfI17RAo3keKXbPzMAbWpK6nDNrdVWKxRtTFUYy3opD0D4NIHYoXo9CcAzSzc2e7eHcHM6zEzK2pz+Vm0NhuzwW4VRrfDGZEJalqeQLi0wkZ1WR8co0D1f7B7xRcLnVVZabyoFBC4Rcj+VOw7qx0uHNYYPVC2iDZ7VJjE8I5LHie+hCqQ2d57yr8dReOgWokGQU47gIp2tkjJ04mAzKqliNVOGWQa/+Qu5inaA9ikb+JBTLnNSDIMSZQpGkSjM3kt8GLcey++m5uH2EHaXAzDKB4ipB24nPBK1luE7QWooTgtai/LygtbjjQTPZQjDHbl4xTQbe7kTvarJFGABujsaOB4QxuJDlN4dz++FgDMw3/TfnqtrBYAQrNDwYWjpM/JuRLs2XLHTWePi9aQSuq94Fdan111xvCHK9gYKVWzAZAbMIsmygdDr703t8kZu9JkA03+9pEou0UsOle6W1QtalmjH+RF4UeqKHFktk9ipWvfCM6apINIlYp0QnGbyyz4S5cW9HUplrw/jU1DVQm57fXyahZPkzoJ6+TEJZLJgxKBKT3Zrs90/PH15ePv0yyhdzYg9a3R6jbQp1KZ0+eYly9qB6rB88QOHMPnFm3W/vDI9Vbve7u6Yv2s3yz1+HM6bnHHjfBuFz7YwZq/ufvl5F/+W//n/OfL2SjGa63kfOmw3LxmhWw6Zx3qhi5JLKCOawG8rasv/kzN8rC4RNjiVsGMGQ9s74aBtzUW59YQXO/Vxy2mLoE2ihdBZ1t/ubzvar3S7AKdvmQ7hCEo9K9yyJx8MjclHpXl+/xYO7+i2F9wLOXnQXJB8j17vWDs7VRKTxjziOJg+topC0hopY5lv32hfHTfLUQSdj+SA0rVpgV/E65aa1NjqRCHyTHBOuGEVcwrd2vcYDSKUwYyr8D2TJVSUQciEegEgAOr6YOON8ut5gyG1TpJ6+37SqRlzW3nC7JVWduBNUF95IjcG2X6ZM8JksKasi0lVE3ZEclOa+rNS9vAzD4XjwxprVa+3qOlFsNFiGg1XQWcTZOnhKxfNCodbN1s+blT5tYVXxNbobF8GO3dFy6M1c1fJEQdQkHiDX5B7Aau0m6uLWYHLNsHipp+Hxl+dSCYvDm6uLnNQY3gmDkt4DhIbZcEWvtsB1UbBhX3BbUZ5sUGb58DU9dDGfenoFcRgQ35+eZmTyPD8OwWJfcGcK/5Y0cq/jogWapNoiOalUre7ToNTuX/993CI9wH8eS/3v4VpuZcHsNmFpKkvptrEc76ei2tu+g7XIhCybewjhCafG7mhxtziRCBc8z5BjK+mBxszGfoRh/xRJKAnS+fOuUgJWCJXxURJ6M7zevU475x1RK0NQbtGUXJ/SyKXM5b3Hh97JY5jFsqlXYaCuD/1OKesdsEZr9e6n34ZEfuLB+iR2KwUPdoMT9XCLMWqyHiNv+nxfj+W0oddzxr2X/CWsc8Nl0txEPs1bN3iW7x0+usLyb7064vonAAD//8IHLlo=" + return "eJzEm89v47oRx+/5KwY5tUDitlcfCuwvYIM2m2Czix6KQkuTI4k1RRIkFT//9w+kZNnWb9u0V6f3bO93PhoOh8Mh8whr3C6Bos7vABx3Apdw/wl1fn8HwNBSw7XjSi7hn3cAAP4rKBQrBd4B2FwZl1AlU54tISXC+k8NCiQWl5AR/xt0jsvMLuG/99aK+we4z53T9/+7A0g5CmaXQfkRJCmwYfGP22qvYlSp6096iPzzy/+jX0CVdIRLCy5HKNAZTv1/EwcbNAiWGqKRQWpUAZ++vH5d1AKHGEcoorQOTcK4XTdf9mGNoPlnQOfYT7unDXMIRN4JF2QlcLHaOrRHv9lxCSWz1hcjaP75sFOFoAoqDQ6sqVs/T5UpiFtCF2AH6ZQjIirgD68YB660yKKy/bTIzkdrR1qORLg8Rqx1lE6PNvWOhgiRWEdc2e+vNW43yrDTXPZS6UKlO+a2JqR4gTRHurYL1IrmkcbumWh4R2O5klNmjSolW7wTUWIk4404BO15ALGH4u1oCI5B2rHZsd0fm/1xNiNiO/pjsdk3bOePRVjVal+MBoQhacrpwiBhyUlZZDgzTcNV/gFvFFxuVJnlunSg0YBFqoZip2bdGO7w5rDB6hm0wbNKJ9oHBNJY43voQq60Pc15V+OpvDQHqOBWC0JxEVbXyBg7cdAZyLJYDWThhkGt/o/UxVpBOxS1/CwUQxxXvSCWEoEsSYUibiC+NRqKsv3tqbhdhB0lw8wQhuwqg7YTnxi0huE6g9ZQzBi0BuX3DVqDOzxoOlsw4sjNM6bOwNsdqV11tggbgJujkeMNwhBciPKbw7n95mAIzBf9N+cqm43BAFYoeDCUdJj4DyNNzdcsVNZ4+LtxBKrKzoQ61/pbrjYWcrWBgsgt6MwCMQhc1lAqnXz1Dl/kYq8eIDtd7ynLFmkp+lP3SimBpE01YfzJelHoiB5alEjMVax64QnTZZEoy2KtEq1g8Mo+Eqa2ezuSUl8bxoemqoCa8Pz5OgrF5e+Aevo2CmWwIFojS3R2a7LvX54/vL5++TzIF3PHHrTaNUZTFGYmidWyszjc/2t2uCt0Y72UPq7LGzxHZMNdnhPYlGV+f5NeQvXy9tlXZmHdlHTffVVptbnuTve5pIe0vN3emBdHHu6pP0B9GNsFVUXBXSKIQ0m3SXHeHPoUVKBW8Ymu6F/7K6NEa7G91OYHLzLbZOs9Zaz3lPPf80ybnfeU3SZqE8tKVU3L6WbVVEQrVbUpYwdyQBys+Wa08l492XDl5OXPnCxBeWC2UMFRuoSrxBDXz17tA0fDJ4jA099e/C6sU173ue2QYd9762nIjL6mf2rXLuFjaNmHhtBA6+XQ6EETLYJVr+ZQzjE83g+bb/i7f1GlMex75WC7qfvOEWz/J3S5xo2357AziBevkj1C56yPYTZFKSj6lE4mUpI7NbuSOLkdf6xfZz9GZqe2/Rml7uSBy08o6+7P7sTi+aW/Lu44By47J/laOWPC6v7V16vob/6vj1Pmq4PXaKarU9dps+FINZrVcJ46bVQQ65JSM+Kwf5lj3QVq6n15gbDJUcKGWOjT3hm/ZOHu2v1GCpx6XeuUwaqmWQiVRT3B/rfK9gfYbYA5Z+qHcAW3NCrdM7c0Hp61Lird29uPeHBXvzhxKeDkpDsj+Ih1nbl20D1I2PS29PQTZ1/IK8vAL8VQWpIhcFn54rgVeIXNaNcHoTWnGLYVr5NuGmuDuweG75xiQgWxEa8aNHa9xgNwITAjIvwfcElFyRByxh7AWgbo6GJkjfPhOtf5F7TymxCpzhjelSgHXNZcK7olVRW4I1RnXheLwbY/Mhrh01kiyyLSLLLtgwcQivq0UnUseWiBDw/eULF6rRPJ1ijWGiTD3ix46XaoyXheKOS6yfx5s9SnDKxKukZ34yTYsjuYDr2Zq1oeSYjKsgfIlXUPYJRyI3lxq/Hcjs9syKGuUM4FM9h/PneWk2rDO2EQ3HvAoiYmzOjVFqgqCtLvC2pKmycb5FneP6f7JvPc1SuIQ4/4fvXUAzvP08chWOwK7kzhH9wO3Og465iQ28qidVyISt2HgVTuL/943KJ9gL8/SvXX/lxueEHMNiFpyiV321iO97uiytu+gjVIGJf1TYlwj7u2O5jcDY4Ewhm3NnNsJD3QkNnYFzX3d2VDSuDOr3elYLBCKLUfJaY2/YfY1ynnvCMqZQjKDZrg6zmFXEpc3rkkfSGPJgZlna/Chrpa9FuprLPARunvjbb2br+wPrFdS8GD3WBFPexiDJqsD4Ru+VcMHZZ5m17PGffG3LdwaB2mSX1Vbp63bvAXCxf46ArNv/XqiOvPAAAA//+0W4ni" } diff --git a/metricbeat/module/ceph/mgr/event_mapping.go b/metricbeat/module/ceph/mgr/event_mapping.go new file mode 100644 index 00000000000..ee173cea3d5 --- /dev/null +++ b/metricbeat/module/ceph/mgr/event_mapping.go @@ -0,0 +1,65 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" +) + +// Request stores either or finished command result. +type Request struct { + HasFailed bool `json:"has_failed"` + Finished []Result `json:"finished"` + Failed []Result `json:"failed"` +} + +// Result stores ceph command output (and status). +type Result struct { + Command string `json:"command"` + Outb string `json:"outb"` + Outs string `json:"outs"` +} + +// UnmarshalResponse method unmarshals the content to the given response object. +func UnmarshalResponse(content []byte, response interface{}) error { + var request Request + err := json.Unmarshal(content, &request) + if err != nil { + return errors.Wrap(err, "could not get request data") + } + + if request.HasFailed { + if len(request.Failed) != 1 { + return errors.New("expected single failed command") + } + return fmt.Errorf("%s: %s", request.Failed[0].Outs, request.Failed[0].Command) + } + + if len(request.Finished) != 1 { + return errors.New("expected single finished command") + } + + err = json.Unmarshal([]byte(request.Finished[0].Outb), response) + if err != nil { + return errors.Wrap(err, "could not get response data") + } + return nil +} diff --git a/metricbeat/module/ceph/mgr/metricset.go b/metricbeat/module/ceph/mgr/metricset.go new file mode 100644 index 00000000000..a982cf706b2 --- /dev/null +++ b/metricbeat/module/ceph/mgr/metricset.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" +) + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +var _ mb.MetricSet = new(MetricSet) + +// NewMetricSet creates an metric set that can be used to build other metricsets that query Ceph mgr API. +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + return &MetricSet{ + base, + http, + }, nil +} + +func (m *MetricSet) WithPrefix(prefix string) *MetricSet { + m.HTTP.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, prefix))) + return m +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json new file mode 100644 index 00000000000..1ca708c45ea --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "ceph": { + "cluster_disk": { + "available": { + "bytes": 0 + }, + "total": { + "bytes": 0 + }, + "used": { + "bytes": 0 + } + } + }, + "event": { + "dataset": "ceph.mgr_cluster_disk", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_cluster_disk" + }, + "service": { + "address": "localhost:8003", + "type": "ceph" + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc new file mode 100644 index 00000000000..00e5d01fd51 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_cluster_disk` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml new file mode 100644 index 00000000000..f6ae58cefaa --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: mgr_cluster_disk + type: group + description: > + see: cluster_disk + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml new file mode 100644 index 00000000000..fdc68400c57 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/request?wait=1" +suffix: json diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json new file mode 100644 index 00000000000..409e19fb217 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json-expected.json new file mode 100644 index 00000000000..64265a15562 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json-expected.json @@ -0,0 +1,3 @@ +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_cluster_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_cluster_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json new file mode 100644 index 00000000000..68f48340128 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "df format=json-pretty", + "outb": "{\n \"stats\": {\n \"total_bytes\": 10737418240,\n \"total_avail_bytes\": 9659875328,\n \"total_used_bytes\": 3801088,\n \"total_used_raw_bytes\": 1077542912,\n \"total_used_raw_ratio\": 0.10035400092601776,\n \"num_osds\": 1,\n \"num_per_pool_osds\": 1\n },\n \"stats_by_class\": {},\n \"pools\": [\n {\n \"name\": \"rbd\",\n \"id\": 1,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_data\",\n \"id\": 2,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_metadata\",\n \"id\": 3,\n \"stats\": {\n \"stored\": 2286,\n \"objects\": 22,\n \"kb_used\": 512,\n \"bytes_used\": 524288,\n \"percent_used\": 5.7465484132990241e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \".rgw.root\",\n \"id\": 4,\n \"stats\": {\n \"stored\": 2398,\n \"objects\": 6,\n \"kb_used\": 384,\n \"bytes_used\": 393216,\n \"percent_used\": 4.3099731556139886e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.control\",\n \"id\": 5,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 8,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.meta\",\n \"id\": 6,\n \"stats\": {\n \"stored\": 736,\n \"objects\": 5,\n \"kb_used\": 256,\n \"bytes_used\": 262144,\n \"percent_used\": 2.873356788768433e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.log\",\n \"id\": 7,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 176,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.buckets.index\",\n \"id\": 8,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 1,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n }\n ]\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "139687220240336", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json new file mode 100644 index 00000000000..bc10caba343 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json @@ -0,0 +1,3 @@ +[ + {"ceph":{"cluster_disk":{"available":{"bytes":9659875328},"total":{"bytes":10737418240},"used":{"bytes":3801088}}},"event":{"dataset":"ceph.mgr_cluster_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_cluster_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_cluster_disk/data.go b/metricbeat/module/ceph/mgr_cluster_disk/data.go new file mode 100644 index 00000000000..cbb69d05416 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/data.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_disk + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type DfResponse struct { + Stats struct { + TotalBytes uint64 `json:"total_bytes"` + TotalAvailableBytes uint64 `json:"total_avail_bytes"` + TotalUsedBytes uint64 `json:"total_used_bytes"` + } `json:"stats"` +} + +func eventMapping(content []byte) (common.MapStr, error) { + var response DfResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + return common.MapStr{ + "used": common.MapStr{ + "bytes": response.Stats.TotalUsedBytes, + }, + "total": common.MapStr{ + "bytes": response.Stats.TotalBytes, + }, + "available": common.MapStr{ + "bytes": response.Stats.TotalAvailableBytes, + }, + }, nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go new file mode 100644 index 00000000000..0cb42828faf --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go @@ -0,0 +1,81 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_disk + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "df" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_cluster_disk", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + *mgr.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := mgr.NewMetricSet(base) + if err != nil { + return nil, err + } + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + event, err := eventMapping(content) + if err != nil { + return err + } + + reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "cluster_disk": event, + }}) + return nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go new file mode 100644 index 00000000000..47f0fa15e4d --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_cluster_disk + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_disk"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go new file mode 100644 index 00000000000..b227f4c3f02 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_disk + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" +) + +func TestDataFiles(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_cluster_disk") +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json new file mode 100644 index 00000000000..14cee62d328 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -0,0 +1,31 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "ceph": { + "cluster_health": { + "overall_status": "HEALTH_OK", + "timechecks": { + "epoch": 3, + "round": { + "status": "finished", + "value": 0 + } + } + } + }, + "event": { + "dataset": "ceph.mgr_cluster_health", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_cluster_health" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc new file mode 100644 index 00000000000..73d74cd225c --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_cluster_health` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml new file mode 100644 index 00000000000..6d627dcb88d --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: mgr_cluster_health + type: group + description: > + see: cluster_health + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/failed.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/failed.json new file mode 100644 index 00000000000..409e19fb217 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/failed.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/status.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/status.json new file mode 100644 index 00000000000..258dbe9239a --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/status.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "status format=json-pretty", + "outb": "{\n \"fsid\": \"aa90baeb-6f42-4c6d-8728-882907da4655\",\n \"health\": {\n \"checks\": {},\n \"status\": \"HEALTH_OK\"\n },\n \"election_epoch\": 3,\n \"quorum\": [\n 0\n ],\n \"quorum_names\": [\n \"42697164830d\"\n ],\n \"quorum_age\": 1200,\n \"monmap\": {\n \"epoch\": 1,\n \"fsid\": \"aa90baeb-6f42-4c6d-8728-882907da4655\",\n \"modified\": \"2020-02-13 12:37:02.294475\",\n \"created\": \"2020-02-13 12:37:02.294475\",\n \"min_mon_release\": 14,\n \"min_mon_release_name\": \"nautilus\",\n \"features\": {\n \"persistent\": [\n \"kraken\",\n \"luminous\",\n \"mimic\",\n \"osdmap-prune\",\n \"nautilus\"\n ],\n \"optional\": []\n },\n \"mons\": [\n {\n \"rank\": 0,\n \"name\": \"42697164830d\",\n \"public_addrs\": {\n \"addrvec\": [\n {\n \"type\": \"v2\",\n \"addr\": \"172.30.0.2:3300\",\n \"nonce\": 0\n }\n ]\n },\n \"addr\": \"172.30.0.2:3300/0\",\n \"public_addr\": \"172.30.0.2:3300/0\"\n }\n ]\n },\n \"osdmap\": {\n \"osdmap\": {\n \"epoch\": 18,\n \"num_osds\": 1,\n \"num_up_osds\": 1,\n \"num_in_osds\": 1,\n \"num_remapped_pgs\": 0\n }\n },\n \"pgmap\": {\n \"pgs_by_state\": [\n {\n \"state_name\": \"active+clean\",\n \"count\": 64\n }\n ],\n \"num_pgs\": 64,\n \"num_pools\": 8,\n \"num_objects\": 218,\n \"data_bytes\": 5420,\n \"bytes_used\": 1077542912,\n \"bytes_avail\": 9659875328,\n \"bytes_total\": 10737418240\n },\n \"fsmap\": {\n \"epoch\": 5,\n \"id\": 1,\n \"up\": 1,\n \"in\": 1,\n \"max\": 1,\n \"by_rank\": [\n {\n \"filesystem_id\": 1,\n \"rank\": 0,\n \"name\": \"demo\",\n \"status\": \"up:active\",\n \"gid\": 4129\n }\n ],\n \"up:standby\": 0\n },\n \"mgrmap\": {\n \"epoch\": 5,\n \"active_gid\": 4100,\n \"active_name\": \"42697164830d\",\n \"active_addrs\": {\n \"addrvec\": [\n {\n \"type\": \"v2\",\n \"addr\": \"172.30.0.2:6800\",\n \"nonce\": 206\n },\n {\n \"type\": \"v1\",\n \"addr\": \"172.30.0.2:6801\",\n \"nonce\": 206\n }\n ]\n },\n \"active_addr\": \"172.30.0.2:6801/206\",\n \"active_change\": \"2020-02-13 12:37:04.535336\",\n \"available\": true,\n \"standbys\": [],\n \"modules\": [\n \"iostat\",\n \"restful\"\n ],\n \"available_modules\": [\n {\n \"name\": \"ansible\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"password\": {\n \"name\": \"password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_url\": {\n \"name\": \"server_url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"username\": {\n \"name\": \"username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"verify_server\": {\n \"name\": \"verify_server\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"balancer\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"active\": {\n \"name\": \"active\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"automatically balance PGs across cluster\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"begin_time\": {\n \"name\": \"begin_time\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"beginning time of day to automatically balance\",\n \"long_desc\": \"This is a time of day in the format HHMM.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"begin_weekday\": {\n \"name\": \"begin_weekday\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0\",\n \"min\": \"0\",\n \"max\": \"7\",\n \"enum_allowed\": [],\n \"desc\": \"Restrict automatic balancing to this day of the week or later\",\n \"long_desc\": \"0 or 7 = Sunday, 1 = Monday, etc.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crush_compat_max_iterations\": {\n \"name\": \"crush_compat_max_iterations\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"25\",\n \"min\": \"1\",\n \"max\": \"250\",\n \"enum_allowed\": [],\n \"desc\": \"maximum number of iterations to attempt optimization\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crush_compat_metrics\": {\n \"name\": \"crush_compat_metrics\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"pgs,objects,bytes\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"metrics with which to calculate OSD utilization\",\n \"long_desc\": \"Value is a list of one or more of \\\"pgs\\\", \\\"objects\\\", or \\\"bytes\\\", and indicates which metrics to use to balance utilization.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crush_compat_step\": {\n \"name\": \"crush_compat_step\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0.5\",\n \"min\": \"0.001\",\n \"max\": \"0.999\",\n \"enum_allowed\": [],\n \"desc\": \"aggressiveness of optimization\",\n \"long_desc\": \".99 is very aggressive, .01 is less aggressive\",\n \"tags\": [],\n \"see_also\": []\n },\n \"end_time\": {\n \"name\": \"end_time\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"2400\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"ending time of day to automatically balance\",\n \"long_desc\": \"This is a time of day in the format HHMM.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"end_weekday\": {\n \"name\": \"end_weekday\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"7\",\n \"min\": \"0\",\n \"max\": \"7\",\n \"enum_allowed\": [],\n \"desc\": \"Restrict automatic balancing to days of the week earlier than this\",\n \"long_desc\": \"0 or 7 = Sunday, 1 = Monday, etc.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"min_score\": {\n \"name\": \"min_score\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"minimum score, below which no optimization is attempted\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"mode\": {\n \"name\": \"mode\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"none\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [\n \"crush-compat\",\n \"none\",\n \"upmap\"\n ],\n \"desc\": \"Balancer mode\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"pool_ids\": {\n \"name\": \"pool_ids\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"pools which the automatic balancing will be limited to\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"60\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to wake up and attempt optimization\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"upmap_max_deviation\": {\n \"name\": \"upmap_max_deviation\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0.01\",\n \"min\": \"0\",\n \"max\": \"1\",\n \"enum_allowed\": [],\n \"desc\": \"deviation below which no optimization is attempted\",\n \"long_desc\": \"If the ratio between the fullest and least-full OSD is below this value then we stop trying to optimize placement.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"upmap_max_iterations\": {\n \"name\": \"upmap_max_iterations\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"10\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"maximum upmap optimization iterations\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"crash\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"retain_interval\": {\n \"name\": \"retain_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"31536000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how long to retain crashes before pruning them\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"warn_recent_interval\": {\n \"name\": \"warn_recent_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"1209600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"time interval in which to warn about recent crashes\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"dashboard\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"ALERTMANAGER_API_HOST\": {\n \"name\": \"ALERTMANAGER_API_HOST\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"AUDIT_API_ENABLED\": {\n \"name\": \"AUDIT_API_ENABLED\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"AUDIT_API_LOG_PAYLOAD\": {\n \"name\": \"AUDIT_API_LOG_PAYLOAD\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ENABLE_BROWSABLE_API\": {\n \"name\": \"ENABLE_BROWSABLE_API\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_cephfs\": {\n \"name\": \"FEATURE_TOGGLE_cephfs\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_iscsi\": {\n \"name\": \"FEATURE_TOGGLE_iscsi\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_mirroring\": {\n \"name\": \"FEATURE_TOGGLE_mirroring\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_rbd\": {\n \"name\": \"FEATURE_TOGGLE_rbd\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_rgw\": {\n \"name\": \"FEATURE_TOGGLE_rgw\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE\": {\n \"name\": \"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_API_PASSWORD\": {\n \"name\": \"GRAFANA_API_PASSWORD\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"admin\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_API_URL\": {\n \"name\": \"GRAFANA_API_URL\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_API_USERNAME\": {\n \"name\": \"GRAFANA_API_USERNAME\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"admin\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_UPDATE_DASHBOARDS\": {\n \"name\": \"GRAFANA_UPDATE_DASHBOARDS\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ISCSI_API_SSL_VERIFICATION\": {\n \"name\": \"ISCSI_API_SSL_VERIFICATION\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"PROMETHEUS_API_HOST\": {\n \"name\": \"PROMETHEUS_API_HOST\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"REST_REQUESTS_TIMEOUT\": {\n \"name\": \"REST_REQUESTS_TIMEOUT\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"45\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_ACCESS_KEY\": {\n \"name\": \"RGW_API_ACCESS_KEY\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_ADMIN_RESOURCE\": {\n \"name\": \"RGW_API_ADMIN_RESOURCE\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"admin\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_HOST\": {\n \"name\": \"RGW_API_HOST\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_PORT\": {\n \"name\": \"RGW_API_PORT\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"80\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_SCHEME\": {\n \"name\": \"RGW_API_SCHEME\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"http\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_SECRET_KEY\": {\n \"name\": \"RGW_API_SECRET_KEY\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_SSL_VERIFY\": {\n \"name\": \"RGW_API_SSL_VERIFY\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_USER_ID\": {\n \"name\": \"RGW_API_USER_ID\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crt_file\": {\n \"name\": \"crt_file\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"jwt_token_ttl\": {\n \"name\": \"jwt_token_ttl\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"28800\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"key_file\": {\n \"name\": \"key_file\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"password\": {\n \"name\": \"password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_addr\": {\n \"name\": \"server_addr\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"0.0.0.0\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_port\": {\n \"name\": \"server_port\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"8080\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ssl\": {\n \"name\": \"ssl\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ssl_server_port\": {\n \"name\": \"ssl_server_port\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"8443\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"standby_behaviour\": {\n \"name\": \"standby_behaviour\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"redirect\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [\n \"error\",\n \"redirect\"\n ],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"standby_error_status_code\": {\n \"name\": \"standby_error_status_code\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"500\",\n \"min\": \"400\",\n \"max\": \"599\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"url_prefix\": {\n \"name\": \"url_prefix\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"username\": {\n \"name\": \"username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"deepsea\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"salt_api_eauth\": {\n \"name\": \"salt_api_eauth\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"sharedsecret\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"salt_api_password\": {\n \"name\": \"salt_api_password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"salt_api_url\": {\n \"name\": \"salt_api_url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"salt_api_username\": {\n \"name\": \"salt_api_username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"devicehealth\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"enable_monitoring\": {\n \"name\": \"enable_monitoring\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"monitor device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"mark_out_threshold\": {\n \"name\": \"mark_out_threshold\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"2419200\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"automatically mark OSD if it may fail before this long\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"pool_name\": {\n \"name\": \"pool_name\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"device_health_metrics\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"name of pool in which to store device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"retention_period\": {\n \"name\": \"retention_period\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"15552000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how long to retain device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"scrape_frequency\": {\n \"name\": \"scrape_frequency\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"86400\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to scrape device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"self_heal\": {\n \"name\": \"self_heal\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"preemptively heal cluster around devices that may fail\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to wake up and check device health\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"warn_threshold\": {\n \"name\": \"warn_threshold\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"7257600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"raise health warning if OSD may fail before this long\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"diskprediction_local\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"predict_interval\": {\n \"name\": \"predict_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"86400\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"influx\",\n \"can_run\": false,\n \"error_string\": \"influxdb python module not found\",\n \"module_options\": {\n \"batch_size\": {\n \"name\": \"batch_size\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"5000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"database\": {\n \"name\": \"database\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"ceph\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"hostname\": {\n \"name\": \"hostname\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"30\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"password\": {\n \"name\": \"password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"port\": {\n \"name\": \"port\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"8086\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ssl\": {\n \"name\": \"ssl\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"false\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"threads\": {\n \"name\": \"threads\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"5\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"username\": {\n \"name\": \"username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"verify_ssl\": {\n \"name\": \"verify_ssl\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"true\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"insights\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"iostat\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"k8sevents\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"ceph_event_retention_days\": {\n \"name\": \"ceph_event_retention_days\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"7\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"Days to hold ceph event information within local cache\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"config_check_secs\": {\n \"name\": \"config_check_secs\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"10\",\n \"min\": \"10\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"interval (secs) to check for cluster configuration changes\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"localpool\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"failure_domain\": {\n \"name\": \"failure_domain\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"host\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"failure domain for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"min_size\": {\n \"name\": \"min_size\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"default min_size for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"num_rep\": {\n \"name\": \"num_rep\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"3\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"default replica count for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"pg_num\": {\n \"name\": \"pg_num\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"128\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"default pg_num for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"prefix\": {\n \"name\": \"prefix\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"name prefix for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"subtree\": {\n \"name\": \"subtree\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"rack\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"CRUSH level for which to create a local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"orchestrator_cli\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"orchestrator\": {\n \"name\": \"orchestrator\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"pg_autoscaler\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"60\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"progress\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"max_completed_events\": {\n \"name\": \"max_completed_events\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"50\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"number of past completed events to remember\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"persist_interval\": {\n \"name\": \"persist_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"5\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to persist completed events\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"prometheus\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"rbd_stats_pools\": {\n \"name\": \"rbd_stats_pools\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rbd_stats_pools_refresh_interval\": {\n \"name\": \"rbd_stats_pools_refresh_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"scrape_interval\": {\n \"name\": \"scrape_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_addr\": {\n \"name\": \"server_addr\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_port\": {\n \"name\": \"server_port\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"rbd_support\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"restful\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"key_file\": {\n \"name\": \"key_file\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_addr\": {\n \"name\": \"server_addr\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_port\": {\n \"name\": \"server_port\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"rook\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"selftest\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"roption1\": {\n \"name\": \"roption1\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"roption2\": {\n \"name\": \"roption2\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"xyz\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption1\": {\n \"name\": \"rwoption1\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption2\": {\n \"name\": \"rwoption2\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption3\": {\n \"name\": \"rwoption3\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption4\": {\n \"name\": \"rwoption4\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption5\": {\n \"name\": \"rwoption5\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption6\": {\n \"name\": \"rwoption6\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"testkey\": {\n \"name\": \"testkey\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"testlkey\": {\n \"name\": \"testlkey\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"testnewline\": {\n \"name\": \"testnewline\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"status\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"telegraf\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"address\": {\n \"name\": \"address\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"unixgram:///tmp/telegraf.sock\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"15\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"telemetry\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"channel_basic\": {\n \"name\": \"channel_basic\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"Share basic cluster information (size, version)\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"channel_crash\": {\n \"name\": \"channel_crash\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"channel_device\": {\n \"name\": \"channel_device\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"channel_ident\": {\n \"name\": \"channel_ident\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"contact\": {\n \"name\": \"contact\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"description\": {\n \"name\": \"description\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"device_url\": {\n \"name\": \"device_url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"https://telemetry.ceph.com/device\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"enabled\": {\n \"name\": \"enabled\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"24\",\n \"min\": \"8\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"last_opt_revision\": {\n \"name\": \"last_opt_revision\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"1\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"leaderboard\": {\n \"name\": \"leaderboard\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"organization\": {\n \"name\": \"organization\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"proxy\": {\n \"name\": \"proxy\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"url\": {\n \"name\": \"url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"https://telemetry.ceph.com/report\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"test_orchestrator\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"volumes\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"zabbix\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"identifier\": {\n \"name\": \"identifier\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"60\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"zabbix_host\": {\n \"name\": \"zabbix_host\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"zabbix_port\": {\n \"name\": \"zabbix_port\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"10051\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"zabbix_sender\": {\n \"name\": \"zabbix_sender\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"/usr/bin/zabbix_sender\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n }\n ],\n \"services\": {\n \"restful\": \"https://42697164830d:8003/\"\n },\n \"always_on_modules\": {\n \"nautilus\": [\n \"balancer\",\n \"crash\",\n \"devicehealth\",\n \"orchestrator_cli\",\n \"progress\",\n \"rbd_support\",\n \"status\",\n \"volumes\"\n ]\n }\n },\n \"servicemap\": {\n \"epoch\": 3,\n \"modified\": \"2020-02-13 12:37:22.548073\",\n \"services\": {\n \"rbd-mirror\": {\n \"daemons\": {\n \"summary\": \"\",\n \"4164\": {\n \"start_epoch\": 3,\n \"start_stamp\": \"2020-02-13 12:37:21.683166\",\n \"gid\": 4164,\n \"addr\": \"172.30.0.2:0/4224599352\",\n \"metadata\": {\n \"arch\": \"x86_64\",\n \"ceph_release\": \"nautilus\",\n \"ceph_version\": \"ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)\",\n \"ceph_version_short\": \"14.2.7\",\n \"cpu\": \"Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz\",\n \"distro\": \"centos\",\n \"distro_description\": \"CentOS Linux 7 (Core)\",\n \"distro_version\": \"7\",\n \"hostname\": \"42697164830d\",\n \"id\": \"admin\",\n \"instance_id\": \"4164\",\n \"kernel_description\": \"#1 SMP Tue Jul 2 22:58:16 UTC 2019\",\n \"kernel_version\": \"4.9.184-linuxkit\",\n \"mem_swap_kb\": \"2097148\",\n \"mem_total_kb\": \"8164244\",\n \"os\": \"Linux\"\n }\n }\n }\n },\n \"rgw\": {\n \"daemons\": {\n \"summary\": \"\",\n \"42697164830d\": {\n \"start_epoch\": 2,\n \"start_stamp\": \"2020-02-13 12:37:19.480410\",\n \"gid\": 4145,\n \"addr\": \"172.30.0.2:0/1483840010\",\n \"metadata\": {\n \"arch\": \"x86_64\",\n \"ceph_release\": \"nautilus\",\n \"ceph_version\": \"ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)\",\n \"ceph_version_short\": \"14.2.7\",\n \"cpu\": \"Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz\",\n \"distro\": \"centos\",\n \"distro_description\": \"CentOS Linux 7 (Core)\",\n \"distro_version\": \"7\",\n \"frontend_config#0\": \"beast endpoint=0.0.0.0:8080\",\n \"frontend_type#0\": \"beast\",\n \"hostname\": \"42697164830d\",\n \"kernel_description\": \"#1 SMP Tue Jul 2 22:58:16 UTC 2019\",\n \"kernel_version\": \"4.9.184-linuxkit\",\n \"mem_swap_kb\": \"2097148\",\n \"mem_total_kb\": \"8164244\",\n \"num_handles\": \"1\",\n \"os\": \"Linux\",\n \"pid\": \"968\",\n \"zone_id\": \"7af69646-d006-4076-bd75-9253e35323b2\",\n \"zone_name\": \"default\",\n \"zonegroup_id\": \"b5760001-d60d-4698-a8a2-2143e35c5543\",\n \"zonegroup_name\": \"default\"\n }\n }\n }\n },\n \"rgw-nfs\": {\n \"daemons\": {\n \"summary\": \"\",\n \"42697164830d\": {\n \"start_epoch\": 3,\n \"start_stamp\": \"2020-02-13 12:37:21.726085\",\n \"gid\": 4168,\n \"addr\": \"172.30.0.2:0/1234614580\",\n \"metadata\": {\n \"arch\": \"x86_64\",\n \"ceph_release\": \"nautilus\",\n \"ceph_version\": \"ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)\",\n \"ceph_version_short\": \"14.2.7\",\n \"cpu\": \"Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz\",\n \"distro\": \"centos\",\n \"distro_description\": \"CentOS Linux 7 (Core)\",\n \"distro_version\": \"7\",\n \"frontend_config#0\": \"rgwlib\",\n \"frontend_type#0\": \"rgw-nfs\",\n \"hostname\": \"42697164830d\",\n \"kernel_description\": \"#1 SMP Tue Jul 2 22:58:16 UTC 2019\",\n \"kernel_version\": \"4.9.184-linuxkit\",\n \"mem_swap_kb\": \"2097148\",\n \"mem_total_kb\": \"8164244\",\n \"num_handles\": \"1\",\n \"os\": \"Linux\",\n \"pid\": \"2004\",\n \"zone_id\": \"7af69646-d006-4076-bd75-9253e35323b2\",\n \"zone_name\": \"default\",\n \"zonegroup_id\": \"b5760001-d60d-4698-a8a2-2143e35c5543\",\n \"zonegroup_name\": \"default\"\n }\n }\n }\n }\n }\n },\n \"progress_events\": {}\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140713918770960", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/time_sync_status.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/time_sync_status.json new file mode 100644 index 00000000000..55dc45fd09b --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/time_sync_status.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "time-sync-status format=json-pretty", + "outb": "{\n \"timechecks\": {\n \"epoch\": 3,\n \"round\": 0,\n \"round_status\": \"finished\"\n }\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140713918770000", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/data.go b/metricbeat/module/ceph/mgr_cluster_health/data.go new file mode 100644 index 00000000000..8dae532bc56 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/data.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_health + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type StatusResponse struct { + Health struct { + Status string `json:"status"` + } `json:"health"` +} + +type TimeSyncStatusResponse struct { + Timechecks struct { + RoundStatus string `json:"round_status"` + Epoch int64 `json:"epoch"` + Round int64 `json:"round"` + } `json:"timechecks"` +} + +func eventMapping(statusContent, timeSyncStatusContent []byte) (common.MapStr, error) { + var statusResponse StatusResponse + err := mgr.UnmarshalResponse(statusContent, &statusResponse) + if err != nil { + return nil, errors.Wrap(err, "could not unmarshal response") + } + + var timeSyncStatusResponse TimeSyncStatusResponse + err = mgr.UnmarshalResponse(timeSyncStatusContent, &timeSyncStatusResponse) + if err != nil { + return nil, errors.Wrap(err, "could not unmarshal response") + } + + return common.MapStr{ + "overall_status": statusResponse.Health.Status, + "timechecks": common.MapStr{ + "epoch": timeSyncStatusResponse.Timechecks.Epoch, + "round": common.MapStr{ + "value": timeSyncStatusResponse.Timechecks.Round, + "status": timeSyncStatusResponse.Timechecks.RoundStatus, + }, + }, + }, nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go new file mode 100644 index 00000000000..b8dc57d2931 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go @@ -0,0 +1,90 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_health + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephStatusPrefix = "status" + cephTimeSyncStatusPrefix = "time-sync-status" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_cluster_health", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + *mgr.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := mgr.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{metricSet}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + m.HTTP.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephStatusPrefix))) + statusContent, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + m.HTTP.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephTimeSyncStatusPrefix))) + timeStatusContent, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + event, err := eventMapping(statusContent, timeStatusContent) + if err != nil { + return err + } + + reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "cluster_health": event, + }}) + return nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go new file mode 100644 index 00000000000..9113992a86b --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -0,0 +1,51 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_cluster_health + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_health"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go new file mode 100644 index 00000000000..255d96cfd7f --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go @@ -0,0 +1,116 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_health + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +type clientRequest struct { + Prefix string `json:"prefix"` +} + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("./_meta/testdata/") + assert.NoError(t, err) + + statusResponse, err := ioutil.ReadFile(absPath + "/status.json") + assert.NoError(t, err) + timeSyncStatusResponse, err := ioutil.ReadFile(absPath + "/time_sync_status.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + + defer r.Body.Close() + var request clientRequest + err := json.NewDecoder(r.Body).Decode(&request) + require.NoError(t, err) + + if request.Prefix == "status" { + w.Write(statusResponse) + } else if request.Prefix == "time-sync-status" { + w.Write(timeSyncStatusResponse) + } + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_health"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].ModuleFields["cluster_health"].(common.MapStr) + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + assert.EqualValues(t, "HEALTH_OK", event["overall_status"]) + + timechecks := event["timechecks"].(common.MapStr) + assert.EqualValues(t, 3, timechecks["epoch"]) + + round := timechecks["round"].(common.MapStr) + assert.EqualValues(t, 0, round["value"]) + assert.EqualValues(t, "finished", round["status"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("./_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/failed.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_health"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json new file mode 100644 index 00000000000..ec4fc55b575 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json @@ -0,0 +1,26 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "ceph": { + "mgr_osd_perf": { + "id": 0, + "stats": { + "commit_latency_ms": 23, + "apply_latency_ms": 23, + "commit_latency_ns": 23000000, + "apply_latency_ns": 23000000 + } + } + }, + "event": { + "dataset": "ceph.mgr_osd_perf", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_osd_perf" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" + } +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc new file mode 100644 index 00000000000..2af82348aa6 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_perf` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml new file mode 100644 index 00000000000..5adb0f64f1d --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml @@ -0,0 +1,21 @@ +- name: mgr_osd_perf + type: group + description: > + OSD performance metrics of Ceph cluster + release: beta + fields: + - name: id + type: long + description: OSD ID + - name: stats.commit_latency_ms + type: long + description: Commit latency in ms + - name: stats.apply_latency_ms + type: long + description: Apply latency in ms + - name: stats.commit_latency_ns + type: long + description: Commit latency in ns + - name: stats.apply_latency_ns + type: long + description: Apply latency in ns diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml new file mode 100644 index 00000000000..fdc68400c57 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/request?wait=1" +suffix: json diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json new file mode 100644 index 00000000000..409e19fb217 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json-expected.json new file mode 100644 index 00000000000..d61d21f6244 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json-expected.json @@ -0,0 +1,3 @@ +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_osd_perf","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_perf","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json new file mode 100644 index 00000000000..8d2dcf75b58 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "osd perf format=json-pretty", + "outb": "{\n \"pg_ready\": true,\n \"osdstats\": {\n \"osd_perf_infos\": [\n {\n \"id\": 1,\n \"perf_stats\": {\n \"commit_latency_ms\": 1,\n \"apply_latency_ms\": 2,\n \"commit_latency_ns\": 3,\n \"apply_latency_ns\": 4\n }\n }\n ]\n }\n\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140301322418768", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json-expected.json new file mode 100644 index 00000000000..f36c0d5ef61 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json-expected.json @@ -0,0 +1,3 @@ +[ + {"ceph":{"mgr_osd_perf":{"id":1,"stats":{"apply_latency_ms":2,"apply_latency_ns":4,"commit_latency_ms":1,"commit_latency_ns":3}}},"event":{"dataset":"ceph.mgr_osd_perf","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_perf","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_perf/data.go b/metricbeat/module/ceph/mgr_osd_perf/data.go new file mode 100644 index 00000000000..18b78ac6114 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/data.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_perf + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type OsdPerfResponse struct { + OsdStats struct { + OsdPerfInfos []struct { + ID int64 `json:"id"` + PerfStats struct { + CommitLatencyMs uint64 `json:"commit_latency_ms"` + ApplyLatencyMs uint64 `json:"apply_latency_ms"` + CommitLatencyNs uint64 `json:"commit_latency_ns"` + ApplyLatencyNs uint64 `json:"apply_latency_ns"` + } `json:"perf_stats"` + } `json:"osd_perf_infos"` + } `json:"osdstats"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response OsdPerfResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + var events []common.MapStr + for _, OsdPerfInfo := range response.OsdStats.OsdPerfInfos { + event := common.MapStr{ + "id": OsdPerfInfo.ID, + "stats": common.MapStr{ + "commit_latency_ms": OsdPerfInfo.PerfStats.CommitLatencyMs, + "apply_latency_ms": OsdPerfInfo.PerfStats.ApplyLatencyMs, + "commit_latency_ns": OsdPerfInfo.PerfStats.CommitLatencyNs, + "apply_latency_ns": OsdPerfInfo.PerfStats.ApplyLatencyNs, + }, + } + events = append(events, event) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go new file mode 100644 index 00000000000..bd8fc20efd7 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_perf + +import ( + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "osd perf" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_perf", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + *mgr.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := mgr.NewMetricSet(base) + if err != nil { + return nil, err + } + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{MetricSetFields: event}) + if !reported { + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go new file mode 100644 index 00000000000..c592f764ed3 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_perf + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_perf"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go new file mode 100644 index 00000000000..9cd253b9d07 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_perf + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" +) + +func TestDataFiles(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_osd_perf") +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json new file mode 100644 index 00000000000..180c8f7a650 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json @@ -0,0 +1,27 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "ceph": { + "mgr_osd_pool_stats": { + "pool_id": "9", + "pool_name": "scbench", + "client_io_rate": { + "read_bytes_sec": "85", + "write_bytes_sec": "802631707", + "read_op_per_sec": "0", + "write_op_per_sec": "336" + } + } + }, + "event": { + "dataset": "ceph.mgr_osd_pool_stats", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_osd_pool_stats" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" + } +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc new file mode 100644 index 00000000000..618cf1bdaab --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_pool_stats` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml new file mode 100644 index 00000000000..b6eabf1d665 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml @@ -0,0 +1,28 @@ +- name: mgr_osd_pool_stats + type: group + description: > + OSD pool stats of Ceph cluster + release: beta + fields: + - name: pool_name + type: keyword + description: Pool name + - name: pool_id + type: long + description: Pool ID + - name: client_io_rate + type: object + description: Client I/O rates + fields: + - name: read_bytes_sec + type: long + keyword: Bytes read per second + - name: write_bytes_sec + type: long + keyword: Bytes written per second + - name: read_op_per_sec + type: long + keyword: Read operations per second + - name: write_op_per_sec + type: long + keyword: Write operations per second diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml new file mode 100644 index 00000000000..fdc68400c57 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/request?wait=1" +suffix: json diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json new file mode 100644 index 00000000000..409e19fb217 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json-expected.json new file mode 100644 index 00000000000..7ec863fab22 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json-expected.json @@ -0,0 +1,3 @@ +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json new file mode 100644 index 00000000000..af0a1ef6a30 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "osd pool stats format=json-pretty", + "outb": "[\n {\n \"pool_name\": \"rbd\",\n \"pool_id\": 1,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": { \"read_bytes_sec\": 1, \"read_op_per_sec\": 2, \"write_bytes_sec\": 3, \"write_op_per_sec\": 4 }\n },\n {\n \"pool_name\": \"cephfs_data\",\n \"pool_id\": 2,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"cephfs_metadata\",\n \"pool_id\": 3,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \".rgw.root\",\n \"pool_id\": 4,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.control\",\n \"pool_id\": 5,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.meta\",\n \"pool_id\": 6,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.log\",\n \"pool_id\": 7,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.buckets.index\",\n \"pool_id\": 8,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n }\n]\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140301322349968", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json-expected.json new file mode 100644 index 00000000000..50146e84c54 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json-expected.json @@ -0,0 +1,10 @@ +[ + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":5,"pool_name":"default.rgw.control"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":7,"pool_name":"default.rgw.log"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":4,"pool_name":".rgw.root"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":3,"pool_name":"cephfs_metadata"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":2,"pool_name":"cephfs_data"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":1,"read_op_per_sec":2,"write_bytes_sec":3,"write_op_per_sec":4},"pool_id":1,"pool_name":"rbd"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":6,"pool_name":"default.rgw.meta"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":8,"pool_name":"default.rgw.buckets.index"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/data.go b/metricbeat/module/ceph/mgr_osd_pool_stats/data.go new file mode 100644 index 00000000000..4a86c6b9327 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/data.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_pool_stats + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type OsdPoolStat struct { + PoolID uint64 `json:"pool_id"` + PoolName string `json:"pool_name"` + ClientIORate struct { + ReadBytesSec uint64 `json:"read_bytes_sec"` + WriteBytesSec uint64 `json:"write_bytes_sec"` + ReadOpPerSec uint64 `json:"read_op_per_sec"` + WriteOpPerSec uint64 `json:"write_op_per_sec"` + } `json:"client_io_rate"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response []OsdPoolStat + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + var events []common.MapStr + for _, stat := range response { + event := common.MapStr{ + "pool_id": stat.PoolID, + "pool_name": stat.PoolName, + "client_io_rate": common.MapStr{ + "read_bytes_sec": stat.ClientIORate.ReadBytesSec, + "write_bytes_sec": stat.ClientIORate.WriteBytesSec, + "read_op_per_sec": stat.ClientIORate.ReadOpPerSec, + "write_op_per_sec": stat.ClientIORate.WriteOpPerSec, + }, + } + events = append(events, event) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go new file mode 100644 index 00000000000..8d864a623e1 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_pool_stats + +import ( + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "osd pool stats" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_pool_stats", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + *mgr.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := mgr.NewMetricSet(base) + if err != nil { + return nil, err + } + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{MetricSetFields: event}) + if !reported { + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go new file mode 100644 index 00000000000..af2d8898fc2 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_pool_stats + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_pool_stats"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go new file mode 100644 index 00000000000..9e3e2772911 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_pool_stats + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" +) + +func TestDataFiles(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_osd_pool_stats") +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json new file mode 100644 index 00000000000..9816f8f0527 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -0,0 +1,27 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "ceph": { + "osd_tree": { + "children": [ + "-2" + ], + "father": "", + "id": -1, + "name": "default", + "type": "root", + "type_id": 10 + } + }, + "event": { + "dataset": "ceph.mgr_osd_tree", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_osd_tree" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" + } +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc new file mode 100644 index 00000000000..440f11414f9 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_tree` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml new file mode 100644 index 00000000000..2be502ea171 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: mgr_osd_tree + type: group + description: > + see: osd_tree + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml new file mode 100644 index 00000000000..fdc68400c57 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/request?wait=1" +suffix: json diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json new file mode 100644 index 00000000000..409e19fb217 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json-expected.json new file mode 100644 index 00000000000..4deb6443f43 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json-expected.json @@ -0,0 +1,3 @@ +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json new file mode 100644 index 00000000000..28f7edd180d --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "osd tree format=json-pretty", + "outb": "{\n \"nodes\": [\n {\n \"children\": [\n -3\n ],\n \"id\": -1,\n \"name\": \"default\",\n \"type\": \"root\",\n \"type_id\": 10\n },\n {\n \"children\": [\n 1,\n 0\n ],\n \"id\": -3,\n \"name\": \"ceph-mon1\",\n \"pool_weights\": {},\n \"type\": \"host\",\n \"type_id\": 1\n },\n {\n \"crush_weight\": 0.048691,\n \"depth\": 2,\n \"device_class\": \"hdd\",\n \"exists\": 1,\n \"id\": 0,\n \"name\": \"osd.0\",\n \"pool_weights\": {},\n \"primary_affinity\": 1.0,\n \"reweight\": 1.0,\n \"status\": \"up\",\n \"type\": \"osd\",\n \"type_id\": 0\n },\n {\n \"crush_weight\": 0.048691,\n \"depth\": 2,\n \"device_class\": \"hdd\",\n \"exists\": 1,\n \"id\": 1,\n \"name\": \"osd.1\",\n \"pool_weights\": {},\n \"primary_affinity\": 1.0,\n \"reweight\": 1.0,\n \"status\": \"up\",\n \"type\": \"osd\",\n \"type_id\": 0\n }\n ],\n \"stray\": []\n }", + "outs": "" + } + ], + "has_failed": false, + "id": "140149339315152", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json new file mode 100644 index 00000000000..c2118414dcb --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json @@ -0,0 +1,6 @@ +[ + {"ceph":{"osd_tree":{"children":["1","0"],"father":"default","id":-3,"name":"ceph-mon1","type":"host","type_id":1}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"osd_tree":{"children":["-3"],"father":"","id":-1,"name":"default","type":"root","type_id":10}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":1,"name":"osd.1","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":0,"name":"osd.0","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_tree/data.go b/metricbeat/module/ceph/mgr_osd_tree/data.go new file mode 100644 index 00000000000..8694c21f90a --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/data.go @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_tree + +import ( + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type OsdTreeResponse struct { + Nodes []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + TypeID int64 `json:"type_id"` + Children []int64 `json:"children"` + CrushWeight float64 `json:"crush_weight"` + Depth int64 `json:"depth"` + Exist int64 `json:"exists"` + PrimaryAffinity float64 `json:"primary_affinity"` + Reweight float64 `json:"reweight"` + Status string `json:"status"` + DeviceClass string `json:"device_class"` + } `json:"nodes"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response OsdTreeResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + nodeList := response.Nodes + + //generate fatherNode and children map + fatherMap := make(map[string]string) + childrenMap := make(map[string]string) + + for _, node := range nodeList { + if node.ID >= 0 { + continue // it's OSD node + } + var childrenList []string + for _, child := range node.Children { + childIDStr := strconv.FormatInt(child, 10) + childrenList = append(childrenList, childIDStr) + fatherMap[childIDStr] = node.Name + } + // generate bucket node's children list + childrenMap[node.Name] = strings.Join(childrenList, ",") + } + + // OSD node list + var events []common.MapStr + for _, node := range nodeList { + nodeInfo := common.MapStr{} + if node.ID < 0 { + // bucket node + nodeInfo["children"] = strings.Split(childrenMap[node.Name], ",") + } else { + // OSD node + nodeInfo["crush_weight"] = node.CrushWeight + nodeInfo["depth"] = node.Depth + nodeInfo["primary_affinity"] = node.PrimaryAffinity + nodeInfo["reweight"] = node.Reweight + nodeInfo["status"] = node.Status + nodeInfo["device_class"] = node.DeviceClass + if node.Exist > 0 { + nodeInfo["exists"] = true + } else { + nodeInfo["exists"] = false + } + } + nodeInfo["id"] = node.ID + nodeInfo["name"] = node.Name + nodeInfo["type"] = node.Type + nodeInfo["type_id"] = node.TypeID + + idStr := strconv.FormatInt(node.ID, 10) + nodeInfo["father"] = fatherMap[idStr] + + events = append(events, nodeInfo) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go new file mode 100644 index 00000000000..49b197f2682 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_tree + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "osd tree" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_tree", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + *mgr.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := mgr.NewMetricSet(base) + if err != nil { + return nil, err + } + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "osd_tree": event, + }}) + if !reported { + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go new file mode 100644 index 00000000000..cedc692e079 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_tree + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_tree"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go new file mode 100644 index 00000000000..d59ac2803e0 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_tree + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" +) + +func TestDataFiles(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_osd_tree") +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json new file mode 100644 index 00000000000..87d03b1e144 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json @@ -0,0 +1,31 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "ceph": { + "pool_disk": { + "id": 0, + "name": "rbd", + "stats": { + "available": { + "bytes": 0 + }, + "objects": 0, + "used": { + "bytes": 0, + "kb": 0 + } + } + } + }, + "event": { + "dataset": "ceph.mgr_pool_disk", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_pool_disk" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" + } +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_pool_disk/_meta/docs.asciidoc new file mode 100644 index 00000000000..983b39f26a9 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_disk` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml new file mode 100644 index 00000000000..2ecc0bdf78d --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: mgr_pool_disk + type: group + description: > + see: pool_disk + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml new file mode 100644 index 00000000000..fdc68400c57 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/request?wait=1" +suffix: json diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json new file mode 100644 index 00000000000..409e19fb217 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json-expected.json new file mode 100644 index 00000000000..2398162a66a --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json-expected.json @@ -0,0 +1,3 @@ +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json new file mode 100644 index 00000000000..68f48340128 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "df format=json-pretty", + "outb": "{\n \"stats\": {\n \"total_bytes\": 10737418240,\n \"total_avail_bytes\": 9659875328,\n \"total_used_bytes\": 3801088,\n \"total_used_raw_bytes\": 1077542912,\n \"total_used_raw_ratio\": 0.10035400092601776,\n \"num_osds\": 1,\n \"num_per_pool_osds\": 1\n },\n \"stats_by_class\": {},\n \"pools\": [\n {\n \"name\": \"rbd\",\n \"id\": 1,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_data\",\n \"id\": 2,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_metadata\",\n \"id\": 3,\n \"stats\": {\n \"stored\": 2286,\n \"objects\": 22,\n \"kb_used\": 512,\n \"bytes_used\": 524288,\n \"percent_used\": 5.7465484132990241e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \".rgw.root\",\n \"id\": 4,\n \"stats\": {\n \"stored\": 2398,\n \"objects\": 6,\n \"kb_used\": 384,\n \"bytes_used\": 393216,\n \"percent_used\": 4.3099731556139886e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.control\",\n \"id\": 5,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 8,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.meta\",\n \"id\": 6,\n \"stats\": {\n \"stored\": 736,\n \"objects\": 5,\n \"kb_used\": 256,\n \"bytes_used\": 262144,\n \"percent_used\": 2.873356788768433e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.log\",\n \"id\": 7,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 176,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.buckets.index\",\n \"id\": 8,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 1,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n }\n ]\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "139687220240336", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json new file mode 100644 index 00000000000..9026c18a168 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json @@ -0,0 +1,10 @@ +[ + {"ceph":{"pool_disk":{"id":7,"name":"default.rgw.log","stats":{"available":{"bytes":9123004416},"objects":176,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":4,"name":".rgw.root","stats":{"available":{"bytes":9123004416},"objects":6,"used":{"bytes":393216,"kb":384}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":1,"name":"rbd","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":2,"name":"cephfs_data","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":3,"name":"cephfs_metadata","stats":{"available":{"bytes":9123004416},"objects":22,"used":{"bytes":524288,"kb":512}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":6,"name":"default.rgw.meta","stats":{"available":{"bytes":9123004416},"objects":5,"used":{"bytes":262144,"kb":256}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":5,"name":"default.rgw.control","stats":{"available":{"bytes":9123004416},"objects":8,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":8,"name":"default.rgw.buckets.index","stats":{"available":{"bytes":9123004416},"objects":1,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_pool_disk/data.go b/metricbeat/module/ceph/mgr_pool_disk/data.go new file mode 100644 index 00000000000..437c164fec9 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/data.go @@ -0,0 +1,67 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_pool_disk + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type DfResponse struct { + Pools []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Stats struct { + BytesUsed uint64 `json:"bytes_used"` + MaxAvail uint64 `json:"max_avail"` + Objects uint64 `json:"objects"` + KbUsed uint64 `json:"kb_used"` + } `json:"stats"` + } `json:"pools"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response DfResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + var events []common.MapStr + for _, Pool := range response.Pools { + event := common.MapStr{ + "name": Pool.Name, + "id": Pool.ID, + "stats": common.MapStr{ + "used": common.MapStr{ + "bytes": Pool.Stats.BytesUsed, + "kb": Pool.Stats.KbUsed, + }, + "available": common.MapStr{ + "bytes": Pool.Stats.MaxAvail, + }, + "objects": Pool.Stats.Objects, + }, + } + + events = append(events, event) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go new file mode 100644 index 00000000000..41b025abf24 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_pool_disk + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "df" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_pool_disk", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + *mgr.MetricSet +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := mgr.NewMetricSet(base) + if err != nil { + return nil, err + } + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "pool_disk": event, + }}) + if !reported { + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go new file mode 100644 index 00000000000..f3b7748263d --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_pool_disk + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_pool_disk"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go new file mode 100644 index 00000000000..ab0c3f03d69 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_pool_disk + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" +) + +func TestDataFiles(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_pool_disk") +} diff --git a/metricbeat/module/ceph/mgrtest/password.go b/metricbeat/module/ceph/mgrtest/password.go new file mode 100644 index 00000000000..c3b646105a2 --- /dev/null +++ b/metricbeat/module/ceph/mgrtest/password.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgrtest + +import ( + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +// GetPassword method returns the password of the given user. Used to authenticate client with Restful API. +func GetPassword(t *testing.T, host, user string) string { + response, err := http.Get(fmt.Sprintf("http://%s/restful-list-keys.json", host)) + require.NoError(t, err) + + defer response.Body.Close() + + accounts := map[string]string{} + json.NewDecoder(response.Body).Decode(&accounts) + return accounts[user] +} diff --git a/metricbeat/module/ceph/osd_df/osd_df_integration_test.go b/metricbeat/module/ceph/osd_df/osd_df_integration_test.go index f2d64e2b573..6735749bd70 100644 --- a/metricbeat/module/ceph/osd_df/osd_df_integration_test.go +++ b/metricbeat/module/ceph/osd_df/osd_df_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go b/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go index 39f78ffea68..69d7e3ed7e3 100644 --- a/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go +++ b/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go b/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go index 07d6b34c5bb..002acbf0c6a 100644 --- a/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go +++ b/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/test_ceph.py b/metricbeat/module/ceph/test_ceph.py index 17c34bb5a14..9fe207e6d65 100644 --- a/metricbeat/module/ceph/test_ceph.py +++ b/metricbeat/module/ceph/test_ceph.py @@ -1,4 +1,5 @@ import os +import requests import sys import time import unittest @@ -8,6 +9,7 @@ import metricbeat +@metricbeat.parameterized_with_supported_versions class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['ceph'] @@ -24,4 +26,61 @@ def test_ceph(self, metricset): """ ceph metricsets tests """ - self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS) + + if not self.old_ceph_version(): + self.skipTest("newer ceph version not supported") + return + self.check_metricset("ceph", metricset, ['http://' + self.compose_host(port='5000/tcp')], self.FIELDS) + + @parameterized.expand([ + "mgr_cluster_disk", + "mgr_cluster_health", + "mgr_osd_perf", + "mgr_osd_pool_stats", + "mgr_osd_tree", + "mgr_pool_disk" + ]) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_ceph_mgr(self, metricset): + """ + ceph metricsets tests + """ + + if self.old_ceph_version(): + self.skipTest("legacy ceph version not supported") + return + + self.render_config_template(modules=[self.get_ceph_mgr_module_config(metricset)]) + proc = self.start_beat(home=self.beat_path) + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings(replace=['SSL/TLS verifications disabled.']) + + output = self.read_output_json() + for evt in output: + assert 'ceph' in evt + self.assert_fields_are_documented(evt) + + def old_ceph_version(self): + if not 'CEPH_CODENAME' in self.COMPOSE_ENV: + return False + + return self.COMPOSE_ENV['CEPH_CODENAME'] == 'jewel' + + def get_ceph_mgr_module_config(self, metricset): + return { + 'name': 'ceph', + 'metricsets': [metricset], + 'period': '1h', + 'hosts': ['https://' + self.compose_host(port='8003/tcp')], + 'username': 'demo', + 'password': self.get_ceph_mgr_password(), + 'extras': { + 'ssl.verification_mode': 'none' + } + } + + def get_ceph_mgr_password(self): + r = requests.get('http://' + self.compose_host(port='5000/tcp') + '/restful-list-keys.json') + keys = r.json() + return keys['demo'] diff --git a/metricbeat/modules.d/ceph-mgr.yml.disabled b/metricbeat/modules.d/ceph-mgr.yml.disabled new file mode 100644 index 00000000000..b844c0df2f7 --- /dev/null +++ b/metricbeat/modules.d/ceph-mgr.yml.disabled @@ -0,0 +1,23 @@ +# Module: ceph +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-ceph.html + +- module: ceph + metricsets: + - mgr_cluster_health + period: 10s + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" + +- module: ceph + metricsets: + - mgr_cluster_disk + - mgr_cluster_health + - mgr_osd_disk + - mgr_osd_perf + # - mgr_osd_pool_stats + # - mgr_osd_tree + period: 1m + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret"