From 57b4a6ffdafb3a02c21cbd038403bcfcf881eeb7 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS Date: Wed, 6 Sep 2023 14:03:18 -0500 Subject: [PATCH] Add remote store and segment replication API updates Signed-off-by: Naarcha-AWS --- _api-reference/cluster-api/cluster-stats.md | 1 + _api-reference/index-apis/stats.md | 2 + _api-reference/nodes-apis/nodes-stats.md | 10 ++ .../remote-store/remote-store-stats-api.md | 114 +++++++++++++++++- 4 files changed, 126 insertions(+), 1 deletion(-) diff --git a/_api-reference/cluster-api/cluster-stats.md b/_api-reference/cluster-api/cluster-stats.md index b717199c24..7f640763b8 100644 --- a/_api-reference/cluster-api/cluster-stats.md +++ b/_api-reference/cluster-api/cluster-stats.md @@ -491,3 +491,4 @@ nodes.network_types | The transport and HTTP networks within the nodes. nodes.discovery_type | The method the nodes use to find other nodes within the cluster. nodes.packaging_types | Information about the nodes' OpenSearch distribution. nodes.ingest | Information about the nodes' ingest pipelines/nodes, if there are any. +total_time_spent | The total amount of download and upload time take across all shards in the cluster when downloading or uploading from the remote store. diff --git a/_api-reference/index-apis/stats.md b/_api-reference/index-apis/stats.md index de887a47d6..77f827e660 100644 --- a/_api-reference/index-apis/stats.md +++ b/_api-reference/index-apis/stats.md @@ -43,7 +43,9 @@ Metric | Description `get` | Get statistics, including missing stats. `indexing` | Indexing statistics. `merge` | Merge statistics. +`nodestats` | Statistics about memory use on nodes for which the index is stored. `query_cache` | Query cache statistics. +`remotesegmentstats` | Statistics about remote storage, including upload and downloads times from the remote store to nodes, the size of each segment in bytes, and the maximum refresh size and times. `refresh` | Refresh statistics. `request_cache` | Shard request cache statistics. `search` | Search statistics, including suggest operation statistics. Search operations can be associated with one or more groups. You can include statistics for custom groups by providing a `groups` parameter, which accepts a comma-separated list of group names. To return statistics for all groups, use `_all`. diff --git a/_api-reference/nodes-apis/nodes-stats.md b/_api-reference/nodes-apis/nodes-stats.md index 77e5f0d095..5490e03571 100644 --- a/_api-reference/nodes-apis/nodes-stats.md +++ b/_api-reference/nodes-apis/nodes-stats.md @@ -739,6 +739,8 @@ search.scroll_current | Integer | The number of scroll operations that are curre search.suggest_total | Integer | The total number of suggest operations. search.suggest_time_in_millis | Integer | The total time for all suggest operations, in milliseconds. search.suggest_current | Integer | The number of suggest operations that are currently running. +total_time_spent | Integer | The cumulative amount of time in seconds both downloading and uploading segments to the remote store across all shards in the node. +total_time_spent_in_millis | The cumulative amount of time in milliseconds both downloading and uploading segments to the remote store across all shards in the node. merges | Object | Statistics about merge operations for the node. merges.current | Integer | The number of merge operations that are currently running. merges.current_docs | Integer | The number of document merges that are currently running. @@ -793,6 +795,14 @@ segments.version_map_memory_in_bytes | Integer | The total amount of memory used segments.fixed_bit_set_memory_in_bytes | Integer | The total amount of memory used by fixed bit sets, in bytes. Fixed bit sets are used for nested objects and join fields. segments.max_unsafe_auto_id_timestamp | Integer | The timestamp for the most recently retired indexing request, in milliseconds since the epoch. segments.file_sizes | Integer | Statistics about the size of the segment files. +max_refresh_time_lag_in_millis | Integer | The maximum refresh lag time in milliseconds across all shards on the node. +max_refresh_size_lag_in_bytes | Integer | The maximum lag size across all the shards on the node. +total_updates.started_bytes | Integer | The amount of segment payload upload attempts to the remote store. +total_uploads.succeeded_bytes | Integer | The amount of successful segment payloads uploaded to the remote store. +total_uploads.failed_bytes | Integer | The amount of failed segment payload uploads to the remote store. +total_downloads.started_bytes | Integer | The amount of segment payload download attempts from the remote store. +total_downloads.succeeded_bytes | Integer | The amount of successful segment payload download attempts from the remote store. +total_downloads.failed_bytes | The amount of failed segment payload download attempts from the remote store. translog | Object | Statistics about transaction log operations for the node. translog.operations | Integer | The number of translog operations. translog.size_in_bytes | Integer | The size of the translog, in bytes. diff --git a/_tuning-your-cluster/availability-and-recovery/remote-store/remote-store-stats-api.md b/_tuning-your-cluster/availability-and-recovery/remote-store/remote-store-stats-api.md index b5b8765738..fff8c9e0e8 100644 --- a/_tuning-your-cluster/availability-and-recovery/remote-store/remote-store-stats-api.md +++ b/_tuning-your-cluster/availability-and-recovery/remote-store/remote-store-stats-api.md @@ -132,11 +132,119 @@ The following table lists the available response fields. |`consecutive_failure_count` | The number of consecutive remote refresh failures since the last success. | |`total_remote_refresh` |The total number of remote refreshes. | |`total_uploads_in_bytes` | The total number of bytes in all uploads to the remote store. | +|`total_download_in_bytes` | The total number of bytes for all downloads from the remote store. | +| `download_size_in_bytes` | The size of the last successful download in bytes. +| `download_speed_in_bytes_per_sec` | The average download speed in bytes per second from the remote store. |`remote_refresh_size_in_bytes.last_successful` |The size of data uploaded in the last successful refresh. | |`remote_refresh_size_in_bytes.moving_avg` |The average size of data (in bytes) uploaded in the last _N_ refreshes. _N_ is defined in `remote_store.segment.pressure.upload_bytes_moving_average_window_size`. For details, see [Remote segment backpressure]({{site.url}}{{site.baseurl}}/tuning-your-cluster/availability-and-recovery/remote-store/remote-segment-backpressure/). | |`upload_latency_in_bytes_per_sec.moving_avg` |The average speed of remote store uploads (in bytes per second) for the last _N_ uploads. _N_ is defined in `remote_store.segment.pressure.upload_bytes_per_sec_moving_average_window_size`. For details, see [Remote segment backpressure]({{site.url}}{{site.baseurl}}/tuning-your-cluster/availability-and-recovery/remote-store/remote-segment-backpressure/). | |`remote_refresh_latency_in_millis.moving_avg` |The average time taken by a single remote refresh during the last _N_ remote refreshes. _N_ is defined in `remote_store.segment.pressure.upload_time_moving_average_window_size`. For details, see [Remote segment backpressure]({{site.url}}{{site.baseurl}}/tuning-your-cluster/availability-and-recovery/remote-store/remote-segment-backpressure/). | +## Remote stote states from all nodes + +Use the following API to get remote store statistics for all nodes related to the index. + +#### Example request + +```json +GET _remotestore/stats/ +``` +{% include copy-curl.html %} + +#### Example response + +
+ + Response + + {: .text-delta } + + +```json +{ + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "indices": { + "remote-index": { + "shards": { + "0": [ + { + "routing": { + "state": "STARTED", + "primary": true, + "node": "EDUazBdFTYK9-Wij2HtTiQ" + }, + "segment": { + "download": {}, + "upload": { + "local_refresh_timestamp_in_millis": 1692264786719, + "remote_refresh_timestamp_in_millis": 1692264786719, + "refresh_time_lag_in_millis": 0, + "refresh_lag": 0, + "bytes_lag": 0, + "backpressure_rejection_count": 0, + "consecutive_failure_count": 0, + "total_syncs_to_remote": { + "started": 333, + "succeeded": 333, + "failed": 0 + }, + "total_uploads_in_bytes": { + "started": 15632029, + "succeeded": 15632029, + "failed": 0 + }, + "remote_refresh_size_in_bytes": { + "last_successful": 18694, + "moving_avg": 88900.45 + }, + "upload_latency_in_bytes_per_sec": { + "moving_avg": 481549.75 + }, + "remote_refresh_latency_in_millis": { + "moving_avg": 92.1 + } + } + } + }, + { + "routing": { + "state": "STARTED", + "primary": false, + "node": "yKBw3ByYSd6KZnx_qOb9NA" + }, + "segment": { + "download": { + "last_sync_timestamp": 1692264787173, + "total_downloads_in_bytes": { + "started": 15631799, + "succeeded": 15631799, + "failed": 0 + }, + "download_size_in_bytes": { + "last_successful": 479, + "moving_avg": 5793.95 + }, + "download_speed_in_bytes_per_sec": { + "moving_avg": 289215.3 + } + }, + "upload": {} + } + } + ] + } + } + } +} +``` +
+ + + ## Remote store stats for a single shard Use the following API to get remote store statistics for a single shard. @@ -204,4 +312,8 @@ Provide the `local` query parameter set to `true` to only fetch the shards prese ```json GET _remotestore/stats/?local=true ``` -{% include copy-curl.html %} \ No newline at end of file +{% include copy-curl.html %} + + + +