From 30ebe6b929422f6a1133b346caf037a3cd1336d7 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Mon, 10 Aug 2020 19:20:53 +0200 Subject: [PATCH] Update vm_compute metricset with more info on guest metrics (#20448) * mofidy doc * update doc * changelog (cherry picked from commit dec042cca15f2fa885f02b262deb7d74e923de6c) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/azure.asciidoc | 1 - .../module/azure/_meta/docs.asciidoc | 1 - .../module/azure/compute_vm/_meta/data.json | 632 +++--------------- .../azure/compute_vm/_meta/docs.asciidoc | 4 + 5 files changed, 102 insertions(+), 537 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e5db74d89c9..8e169260d46 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -362,6 +362,7 @@ field. You can revert this change by configuring tags for the module and omittin - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185] - Add required option for `metrics` in app_insights. {pull}20406[20406] - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403] +- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448] *Packetbeat* diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index e4cd4835473..4db38120041 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -16,7 +16,6 @@ The azure monitor metrics are numerical values that describe some aspect of a sy The azure module will periodically retrieve the azure monitor metrics using the Azure REST APIs as MetricList. Additional azure API calls will be executed in order to retrieve information regarding the resources targeted by the user. -The azure module mericsets are `monitor`, `compute_vm` and `compute_vm_scaleset` IMPORTANT: Extra Azure charges on metric queries may be generated by this module. Please see <> for more details. diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index e828c89e41a..b0f76ecb623 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -8,7 +8,6 @@ The azure monitor metrics are numerical values that describe some aspect of a sy The azure module will periodically retrieve the azure monitor metrics using the Azure REST APIs as MetricList. Additional azure API calls will be executed in order to retrieve information regarding the resources targeted by the user. -The azure module mericsets are `monitor`, `compute_vm` and `compute_vm_scaleset` IMPORTANT: Extra Azure charges on metric queries may be generated by this module. Please see <> for more details. diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 5858cf0407e..960eb88cf7e 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -1,569 +1,131 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "azure": { - "compute_vm": { - "asp_net_application_restarts": { - "avg": 0 - }, - "asp_net_applications_anonymous_requests": { - "avg": 0 - }, - "asp_net_applications_anonymous_requests_per_sec": { - "avg": 0 - }, - "asp_net_applications_cache_api_entries": { - "avg": 0 - }, - "asp_net_applications_cache_api_hit_ratio": { - "avg": 0 - }, - "asp_net_applications_cache_api_hits": { - "avg": 0 - }, - "asp_net_applications_cache_api_misses": { - "avg": 0 - }, - "asp_net_applications_cache_api_turnover_rate": { - "avg": 0 - }, - "asp_net_applications_cache_total_entries": { - "avg": 0 - }, - "asp_net_applications_cache_total_hit_ratio": { - "avg": 0 - }, - "asp_net_applications_cache_total_hits": { - "avg": 0 - }, - "asp_net_applications_cache_total_misses": { - "avg": 0 - }, - "asp_net_applications_cache_total_turnover_rate": { - "avg": 0 - }, - "asp_net_applications_compilations_total": { - "avg": 0 - }, - "asp_net_applications_debugging_requests": { - "avg": 0 - }, - "asp_net_applications_errors_during_compilation": { - "avg": 0 - }, - "asp_net_applications_errors_during_execution": { - "avg": 0 - }, - "asp_net_applications_errors_during_preprocessing": { - "avg": 0 - }, - "asp_net_applications_errors_total": { - "avg": 0 - }, - "asp_net_applications_errors_total_per_sec": { - "avg": 0 - }, - "asp_net_applications_errors_unhandled_during_execution": { - "avg": 0 - }, - "asp_net_applications_errors_unhandled_during_execution_per_sec": { - "avg": 0 - }, - "asp_net_applications_output_cache_entries": { - "avg": 0 - }, - "asp_net_applications_output_cache_hit_ratio": { - "avg": 0 - }, - "asp_net_applications_output_cache_hits": { - "avg": 0 - }, - "asp_net_applications_output_cache_misses": { - "avg": 0 - }, - "asp_net_applications_output_cache_turnover_rate": { - "avg": 0 - }, - "asp_net_applications_pipeline_instance_count": { - "avg": 0 - }, - "asp_net_applications_request_bytes_in_total": { - "avg": 0 - }, - "asp_net_applications_request_bytes_out_total": { - "avg": 0 - }, - "asp_net_applications_requests_executing": { - "avg": 0 - }, - "asp_net_applications_requests_failed": { - "avg": 0 - }, - "asp_net_applications_requests_in_application_queue": { - "avg": 0 - }, - "asp_net_applications_requests_not_authorized": { - "avg": 0 - }, - "asp_net_applications_requests_not_found": { - "avg": 0 - }, - "asp_net_applications_requests_per_sec": { - "avg": 0 - }, - "asp_net_applications_requests_succeeded": { - "avg": 0 - }, - "asp_net_applications_requests_timed_out": { - "avg": 0 - }, - "asp_net_applications_requests_total": { - "avg": 0 - }, - "asp_net_applications_running": { - "avg": 0 - }, - "asp_net_applications_sessions_abandoned": { - "avg": 0 - }, - "asp_net_applications_sessions_active": { - "avg": 0 - }, - "asp_net_applications_sessions_timed_out": { - "avg": 0 - }, - "asp_net_applications_sessions_total": { - "avg": 0 - }, - "asp_net_applications_transactions_aborted": { - "avg": 0 - }, - "asp_net_applications_transactions_committed": { - "avg": 0 - }, - "asp_net_applications_transactions_pending": { - "avg": 0 - }, - "asp_net_applications_transactions_per_sec": { - "avg": 0 - }, - "asp_net_applications_transactions_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_anonymous_requests": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_anonymous_requests_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_entries": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_hit_ratio": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_hits": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_misses": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_api_turnover_rate": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_entries": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_hit_ratio": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_hits": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_misses": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_cache_total_turnover_rate": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_compilations_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_debugging_requests": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_during_compilation": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_during_execution": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_during_preprocessing": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_total_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_unhandled_during_execution": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_errors_unhandled_during_execution_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_entries": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_hit_ratio": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_hits": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_misses": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_output_cache_turnover_rate": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_pipeline_instance_count": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_request_bytes_in_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_request_bytes_out_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_executing": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_failed": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_in_application_queue": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_not_authorized": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_not_found": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_succeeded": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_timed_out": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_requests_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_abandoned": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_active": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_timed_out": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_sessions_total": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_aborted": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_committed": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_pending": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_per_sec": { - "avg": 0 - }, - "asp_net_apps_v4_0_30319_transactions_total": { - "avg": 0 - }, - "asp_net_request_execution_time": { - "avg": 0 - }, - "asp_net_request_wait_time": { - "avg": 0 - }, - "asp_net_requests_current": { - "avg": 0 - }, - "asp_net_requests_disconnected": { - "avg": 0 - }, - "asp_net_requests_queued": { - "avg": 0 - }, - "asp_net_requests_rejected": { - "avg": 0 - }, - "asp_net_v4_0_30319_application_restarts": { - "avg": 0 - }, - "asp_net_v4_0_30319_applications_running": { - "avg": 0 - }, - "asp_net_v4_0_30319_request_execution_time": { - "avg": 0 - }, - "asp_net_v4_0_30319_request_wait_time": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_current": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_disconnected": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_queued": { - "avg": 0 - }, - "asp_net_v4_0_30319_requests_rejected": { - "avg": 0 - }, - "asp_net_v4_0_30319_worker_process_restarts": { - "avg": 0 - }, - "asp_net_v4_0_30319_worker_processes_running": { - "avg": 0 - }, - "asp_net_worker_process_restarts": { - "avg": 0 - }, - "asp_net_worker_processes_running": { - "avg": 0 - }, - "logical_disk_avg_disk_queue_length": { - "avg": 0 - }, - "logical_disk_avg_disk_read_queue_length": { - "avg": 0 - }, - "logical_disk_avg_disk_sec_per_read": { - "avg": 0 - }, - "logical_disk_avg_disk_sec_per_transfer": { - "avg": 0 - }, - "logical_disk_avg_disk_sec_per_write": { - "avg": 0 - }, - "logical_disk_avg_disk_write_queue_length": { - "avg": 0 - }, - "logical_disk_disk_bytes_per_sec": { - "avg": 449380 - }, - "logical_disk_disk_read_bytes_per_sec": { - "avg": 4783 - }, - "logical_disk_disk_reads_per_sec": { - "avg": 0 - }, - "logical_disk_disk_transfers_per_sec": { - "avg": 6.6 - }, - "logical_disk_disk_write_bytes_per_sec": { - "avg": 444596.8 - }, - "logical_disk_disk_writes_per_sec": { - "avg": 6.4 - }, - "logical_disk_free_megabytes": { - "avg": 10433 - }, - "logical_disk_pct_disk_read_time": { - "avg": 0 - }, - "logical_disk_pct_disk_time": { - "avg": 1.6 - }, - "logical_disk_pct_disk_write_time": { - "avg": 1.6 - }, - "logical_disk_pct_free_space": { - "avg": 22 - }, - "logical_disk_pct_idle_time": { - "avg": 98.6 - }, - "memory_available_bytes": { - "avg": 2951829094.4 - }, - "memory_cache_bytes": { - "avg": 60126003.2 - }, - "memory_committed_bytes": { - "avg": 8922399539.2 - }, - "memory_page_faults_per_sec": { - "avg": 1484.2 - }, - "memory_pages_per_sec": { - "avg": 6 - }, - "memory_pct_committed_bytes_in_use": { - "avg": 82.4 - }, - "memory_pool_nonpaged_bytes": { - "avg": 204028313.6 - }, - "memory_pool_paged_bytes": { - "avg": 359613235.2 - }, - "network_interface_bytes_received_per_sec": { - "avg": 7922.333333333333 - }, - "network_interface_bytes_sent_per_sec": { - "avg": 60510.666666666664 - }, - "network_interface_bytes_total_per_sec": { - "avg": 68433.26666666666 - }, - "network_interface_packets_outbound_errors": { - "avg": 0 - }, - "network_interface_packets_per_sec": { - "avg": 43.4 - }, - "network_interface_packets_received_errors": { - "avg": 0 - }, - "network_interface_packets_received_per_sec": { - "avg": 27.533333333333335 + "@timestamp":"2020-08-05T10:36:00.000Z", + "cloud":{ + "provider":"azure", + "region":"westeurope", + "instance":{ + "id":"/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", + "name":"obstestmemleak" + }, + "machine":{ + "type":"Standard_B2ms" + } + }, + "event":{ + "dataset":"azure.compute_vm", + "module":"azure", + "duration":5364973100 + }, + "metricset":{ + "name":"compute_vm", + "period":300000 + }, + "azure":{ + "timegrain":"PT5M", + "resource":{ + "type":"Microsoft.Compute/virtualMachines", + "group":"obs-infrastructure", + "tags":{ + "vmtest":"value1, value 2", + "vmtest1":"value3" + } + }, + "subscription_id":"fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "namespace":"Microsoft.Compute/virtualMachines", + "compute_vm":{ + "outbound_flows":{ + "avg":291.2 }, - "network_interface_packets_sent_per_sec": { - "avg": 15.733333333333333 + "outbound_flows_maximum_creation_rate":{ + "avg":6 }, - "process_handle_count": { - "avg": 119553 + "inbound_flows_maximum_creation_rate":{ + "avg":6 }, - "process_thread_count": { - "avg": 1705.8 + "os_per_disk_read_bytes_per_sec":{ + "avg":341.35875 }, - "process_working_set": { - "avg": 5566961254.4 + "per_disk_read_bytes_per_sec":{ + "avg":0 }, - "process_working_set_private": { - "avg": 4168472985.6 + "os_disk_queue_depth":{ + "avg":0 }, - "processor_information_pct_privileged_time": { - "avg": 2.6 + "os_per_disk_qd":{ + "avg":0 }, - "processor_information_pct_processor_time": { - "avg": 6.4 + "os_disk_read_bytes_per_sec":{ + "avg":341.35875 }, - "processor_information_pct_user_time": { - "avg": 3.4 + "network_in":{ + "total":3933481.0 }, - "processor_information_processor_frequency": { - "avg": 2295 + "network_out":{ + "total":7169752.0 }, - "sqlserver_buffer_manager_buffer_cache_hit_ratio": { - "avg": 100 + "os_disk_write_operations_per_sec":{ + "avg":4.994285714285715 }, - "sqlserver_buffer_manager_checkpoint_pages_per_sec": { - "avg": 0 + "percentage_cpu":{ + "avg":8.780625 }, - "sqlserver_buffer_manager_database_pages": { - "avg": 1483 + "cpu_credits_remaining":{ + "avg":864 }, - "sqlserver_buffer_manager_lazy_writes_per_sec": { - "avg": 0 + "network_out_total":{ + "total":9249694.0 }, - "sqlserver_buffer_manager_page_reads_per_sec": { - "avg": 0 + "disk_read_operations_per_sec":{ + "avg":0.022857142857142857 }, - "sqlserver_buffer_manager_page_writes_per_sec": { - "avg": 0 + "disk_write_bytes":{ + "total":1.759878456E7 }, - "sqlserver_general_statistics_user_connections": { - "avg": 1 + "os_disk_write_bytes_per_sec":{ + "avg":73050.70375 }, - "sqlserver_memory_manager_memory_grants_pending": { - "avg": 0 + "cpu_credits_consumed":{ + "avg":0.175 }, - "sqlserver_memory_manager_total_server_memory": { - "avg": 317976 + "per_disk_qd":{ + "avg":0 }, - "sqlserver_sql_statistics_batch_requests_per_sec": { - "avg": 0 + "disk_write_operations_per_sec":{ + "avg":2.5221428571428572 }, - "sqlserver_sql_statistics_sql_compilations_per_sec": { - "avg": 0 + "os_per_disk_write_operations_per_sec":{ + "avg":4.994285714285715 }, - "sqlserver_sql_statistics_sql_re-compilations_per_sec": { - "avg": 0 + "per_disk_write_operations_per_sec":{ + "avg":0.049999999999999996 }, - "system_context_switches_per_sec": { - "avg": 7782.6 + "inbound_flows":{ + "avg":291.2 }, - "system_processes": { - "avg": 114 + "os_disk_read_operations_per_sec":{ + "avg":0.045714285714285714 }, - "system_processor_queue_length": { - "avg": 0.2 + "disk_read_bytes":{ + "total":81926.52 }, - "system_system_up_time": { - "avg": 12530764 + "network_in_total":{ + "total":4920267.0 }, - "web_service_bytes_total_per_sec": { - "avg": 0 + "per_disk_read_operations_per_sec":{ + "avg":0 }, - "web_service_current_connections": { - "avg": 0 + "per_disk_write_bytes_per_sec":{ + "avg":277.56 }, - "web_service_isapi_extension_requests_per_sec": { - "avg": 0 + "os_per_disk_write_bytes_per_sec":{ + "avg":73050.70375 }, - "web_service_total_method_requests_per_sec": { - "avg": 0 + "os_per_disk_read_operations_per_sec":{ + "avg":0.045714285714285714 } - }, - "namespace": "Azure.VM.Windows.GuestMetrics", - "resource": { - "group": "obs-infrastructure", - "tags": { - "vmtest": "value1, value 2", - "vmtest1": "value3" - }, - "type": "Microsoft.Compute/virtualMachines" - }, - "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", - "timegrain": "PT5M" - }, - "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", - "name": "obstestmemleak" - }, - "machine": { - "type": "Standard_B2ms" - }, - "provider": "azure", - "region": "westeurope" - }, - "event": { - "dataset": "azure.compute_vm", - "duration": 115000, - "module": "azure" - }, - "metricset": { - "name": "compute_vm", - "period": 10000 + } }, - "service": { - "type": "azure" + "service":{ + "type":"azure" } } diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc index e704e9ec664..f30db5e06d8 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc @@ -16,4 +16,8 @@ If none of the options are entered then all virtual machine inside the subscript For each metric the primary aggregation assigned will be retrieved. A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. +To populate the VM Guest Metrics Overview dashboard, users will have to enable the Virtual Machine Guest namespace for the monitored resources. +More on sending guest OS metrics to Azure Monitor here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/collect-custom-metrics-guestos-resource-manager-vm. + +image::./images/metricbeat-azure-vm-guestmetrics-overview.png[]