Skip to content

Commit

Permalink
Allow collection of jvm gc section in node-stats telemetry device
Browse files Browse the repository at this point in the history 
We already collect the mem section under node-stats/jvm but GC stats
can also be very useful for understanding the jvm memory behavior during
a benchmark.

Add `node-stats-include-gc` telemetry parameter (enabled by default) and
add it to docs.

Relates #644
  • Loading branch information
@dliappis
dliappis authored Feb 12, 2019
1 parent 31d5bfb commit 524b6fa
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 0 deletions.
3 changes: 3 additions & 0 deletions docs/telemetry.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ The node-stats telemetry device regularly calls the `cluster node-stats API <htt
* Indices stats (key ``indices`` in the node-stats API)
* Thread pool stats (key ``jvm.thread_pool`` in the node-stats API)
* JVM buffer pool stats (key ``jvm.buffer_pools`` in the node-stats API)
* JVM gc stats (key ``jvm.gc`` in the node-stats API)
* JVM mem stats (key ``jvm.mem`` in the node-stats API)
* Circuit breaker stats (key ``breakers`` in the node-stats API)
* Network-related stats (key ``transport`` in the node-stats API)
* Process cpu stats (key ``process.cpu`` in the node-stats API)
Expand All @@ -98,6 +100,7 @@ Supported telemetry parameters:
* ``node-stats-include-thread-pools`` (default: ``true``): A boolean indicating whether thread pool stats should be included.
* ``node-stats-include-buffer-pools`` (default: ``true``): A boolean indicating whether buffer pool stats should be included.
* ``node-stats-include-breakers`` (default: ``true``): A boolean indicating whether circuit breaker stats should be included.
* ``node-stats-include-gc`` (default: ``true``): A boolean indicating whether JVM gc stats should be included.
* ``node-stats-include-mem`` (default: ``true``): A boolean indicating whether JVM heap stats should be included.
* ``node-stats-include-network`` (default: ``true``): A boolean indicating whether network-related stats should be included.
* ``node-stats-include-process`` (default: ``true``): A boolean indicating whether process cpu stats should be included.
Expand Down
6 changes: 6 additions & 0 deletions esrally/mechanic/telemetry.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,7 @@ def __init__(self, telemetry_params, cluster_name, client, metrics_store):
self.include_network = telemetry_params.get("node-stats-include-network", True)
self.include_process = telemetry_params.get("node-stats-include-process", True)
self.include_mem_stats = telemetry_params.get("node-stats-include-mem", True)
self.include_gc_stats = telemetry_params.get("node-stats-include-gc", True)
self.client = client
self.metrics_store = metrics_store
self.metrics_store_meta_data = {"cluster": cluster_name}
Expand All @@ -659,6 +660,8 @@ def record(self):
collected_node_stats.update(self.jvm_buffer_pool_stats(node_name, node_stats))
if self.include_mem_stats:
collected_node_stats.update(self.jvm_mem_stats(node_name, node_stats))
if self.include_gc_stats:
collected_node_stats.update(self.jvm_gc_stats(node_name, node_stats))
if self.include_network:
collected_node_stats.update(self.network_stats(node_name, node_stats))
if self.include_process:
Expand Down Expand Up @@ -715,6 +718,9 @@ def jvm_buffer_pool_stats(self, node_name, node_stats):
def jvm_mem_stats(self, node_name, node_stats):
return self.flatten_stats_fields(prefix="jvm_mem", stats=node_stats["jvm"]["mem"])

def jvm_gc_stats(self, node_name, node_stats):
return self.flatten_stats_fields(prefix="jvm_gc", stats=node_stats["jvm"]["gc"])

def network_stats(self, node_name, node_stats):
return self.flatten_stats_fields(prefix="transport", stats=node_stats.get("transport"))

Expand Down
48 changes: 48 additions & 0 deletions tests/mechanic/telemetry_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -989,6 +989,18 @@ class NodeStatsRecorderTests(TestCase):
"peak_max_in_bytes": 469368832
}
}
},
"gc": {
"collectors": {
"young": {
"collection_count": 3,
"collection_time_in_millis": 309
},
"old": {
"collection_count": 2,
"collection_time_in_millis": 229
}
}
}
},
"process": {
Expand Down Expand Up @@ -1090,6 +1102,10 @@ class NodeStatsRecorderTests(TestCase):
"jvm_mem_pools_old_max_in_bytes": 469368832,
"jvm_mem_pools_old_peak_used_in_bytes": 52336480,
"jvm_mem_pools_old_peak_max_in_bytes": 469368832,
"jvm_gc_collectors_young_collection_count": 3,
"jvm_gc_collectors_young_collection_time_in_millis": 309,
"jvm_gc_collectors_old_collection_count": 2,
"jvm_gc_collectors_old_collection_time_in_millis": 229,
"process_cpu_percent": 10,
"process_cpu_total_in_millis": 56520,
"breakers_parent_limit_size_in_bytes": 726571417,
Expand Down Expand Up @@ -1274,6 +1290,18 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc):
"peak_max_in_bytes": 469368832
}
}
},
"gc": {
"collectors": {
"young": {
"collection_count": 3,
"collection_time_in_millis": 309
},
"old": {
"collection_count": 2,
"collection_time_in_millis": 229
}
}
}
},
"process": {
Expand Down Expand Up @@ -1395,6 +1423,10 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc):
"jvm_mem_pools_young_peak_max_in_bytes": 139591680,
"jvm_mem_pools_young_peak_used_in_bytes": 139591680,
"jvm_mem_pools_young_used_in_bytes": 66378576,
"jvm_gc_collectors_young_collection_count": 3,
"jvm_gc_collectors_young_collection_time_in_millis": 309,
"jvm_gc_collectors_old_collection_count": 2,
"jvm_gc_collectors_old_collection_time_in_millis": 229,
"transport_rx_count": 77,
"transport_rx_size_in_bytes": 98723498,
"transport_server_open": 12,
Expand Down Expand Up @@ -1534,6 +1566,18 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu
"peak_max_in_bytes": 469368832
}
}
},
"gc": {
"collectors": {
"young": {
"collection_count": 3,
"collection_time_in_millis": 309
},
"old": {
"collection_count": 2,
"collection_time_in_millis": 229
}
}
}
},
"process": {
Expand Down Expand Up @@ -1630,6 +1674,10 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu
"jvm_mem_pools_young_peak_max_in_bytes": 139591680,
"jvm_mem_pools_young_peak_used_in_bytes": 139591680,
"jvm_mem_pools_young_used_in_bytes": 66378576,
"jvm_gc_collectors_young_collection_count": 3,
"jvm_gc_collectors_young_collection_time_in_millis": 309,
"jvm_gc_collectors_old_collection_count": 2,
"jvm_gc_collectors_old_collection_time_in_millis": 229,
"transport_rx_count": 77,
"transport_rx_size_in_bytes": 98723498,
"transport_server_open": 12,
Expand Down

0 comments on commit 524b6fa

Please sign in to comment.