From d1e688a7196822b8a113f77e1ab96da17593c2ec Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:15:50 +0530 Subject: [PATCH 01/36] Adding Segment download stats to remotestore stats API Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../RemoteStoreBackpressureIT.java | 2 +- .../remotestore/RemoteStoreStatsIT.java | 117 ++++++++- .../remotestore/stats/RemoteStoreStats.java | 162 +++++++++--- .../stats/RemoteStoreStatsResponse.java | 90 ++++++- .../TransportRemoteStoreStatsAction.java | 8 +- .../common/settings/ClusterSettings.java | 3 + .../RemoteRefreshSegmentPressureService.java | 15 ++ .../RemoteRefreshSegmentPressureSettings.java | 78 ++++++ .../remote/RemoteRefreshSegmentTracker.java | 239 +++++++++++++++++- .../opensearch/index/shard/IndexShard.java | 49 +++- .../stats/RemoteStoreStatsResponseTests.java | 27 +- .../stats/RemoteStoreStatsTestHelper.java | 194 +++++++++----- .../stats/RemoteStoreStatsTests.java | 67 +++-- .../RemoteRefreshSegmentTrackerTests.java | 110 ++++++-- 14 files changed, 973 insertions(+), 188 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index 64d5f06f061a9..50e09b2830c25 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -119,7 +119,7 @@ private RemoteRefreshSegmentTracker.Stats stats() { String shardId = "0"; RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); - List matches = Arrays.stream(response.getShards()) + List matches = Arrays.stream(response.getRemoteStoreStats()) .filter(stat -> indexShardId.equals(stat.getStats().shardId.toString())) .collect(Collectors.toList()); assertEquals(1, matches.size()); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 0ea87d106c14e..365b975f088e1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -14,7 +14,9 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.UUIDs; +import org.opensearch.common.settings.Settings; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.test.OpenSearchIntegTestCase; @@ -23,6 +25,9 @@ import java.util.Locale; import java.util.stream.Collectors; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 3) public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { @@ -50,14 +55,41 @@ public void testStatsResponseFromAllNodes() { for (String node : nodes) { RemoteStoreStatsResponse response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); assertTrue(response.getSuccessfulShards() > 0); - assertTrue(response.getShards() != null && response.getShards().length != 0); + assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); - List matches = Arrays.stream(response.getShards()) + List matches = Arrays.stream(response.getRemoteStoreStats()) .filter(stat -> indexShardId.equals(stat.getStats().shardId.toString())) .collect(Collectors.toList()); assertEquals(1, matches.size()); RemoteRefreshSegmentTracker.Stats stats = matches.get(0).getStats(); - assertResponseStats(stats); + validateUploadStats(stats); + } + + // Step 3 - Enable replicas on the existing indices and ensure that download + // stats are being populated as well + assertAcked( + client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)) + ); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + for (String node : nodes) { + RemoteStoreStatsResponse response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); + assertTrue(response.getSuccessfulShards() > 0); + assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); + final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); + List matches = Arrays.stream(response.getRemoteStoreStats()) + .filter(stat -> indexShardId.equals(stat.getStats().shardId.toString())) + .collect(Collectors.toList()); + assertEquals(2, matches.size()); + for (RemoteStoreStats stat: matches) { + ShardRouting routing = stat.getShardRouting(); + RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); + if (routing.primary()) { + validateUploadStats(stats); + } else { + validateDownloadStats(stats); + } + } } } @@ -79,10 +111,34 @@ public void testStatsResponseAllShards() { .cluster() .prepareRemoteStoreStats(INDEX_NAME, null); RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); - assertTrue(response.getSuccessfulShards() == 3); - assertTrue(response.getShards() != null && response.getShards().length == 3); - RemoteRefreshSegmentTracker.Stats stats = response.getShards()[0].getStats(); - assertResponseStats(stats); + assertEquals(3, response.getSuccessfulShards()); + assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 3); + RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); + validateUploadStats(stats); + + // Step 3 - Enable replicas on the existing indices and ensure that download + // stats are being populated as well + assertAcked( + client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)) + ); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + remoteStoreStatsRequestBuilder = client(node).admin() + .cluster() + .prepareRemoteStoreStats(INDEX_NAME, null); + response = remoteStoreStatsRequestBuilder.get(); + assertEquals(6, response.getSuccessfulShards()); + assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 6); + for (RemoteStoreStats stat: response.getRemoteStoreStats()) { + ShardRouting routing = stat.getShardRouting(); + stats = stat.getStats(); + if (routing.primary()) { + validateUploadStats(stats); + } else { + validateDownloadStats(stats); + } + } + } public void testStatsResponseFromLocalNode() { @@ -105,10 +161,33 @@ public void testStatsResponseFromLocalNode() { .prepareRemoteStoreStats(INDEX_NAME, null); remoteStoreStatsRequestBuilder.setLocal(true); RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); - assertTrue(response.getSuccessfulShards() == 1); - assertTrue(response.getShards() != null && response.getShards().length == 1); - RemoteRefreshSegmentTracker.Stats stats = response.getShards()[0].getStats(); - assertResponseStats(stats); + assertEquals(1, response.getSuccessfulShards()); + assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 1); + RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); + validateUploadStats(stats); + } + assertAcked( + client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)) + ); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + for (String node : nodes) { + RemoteStoreStatsRequestBuilder remoteStoreStatsRequestBuilder = client(node).admin() + .cluster() + .prepareRemoteStoreStats(INDEX_NAME, null); + remoteStoreStatsRequestBuilder.setLocal(true); + RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); + assertTrue(response.getSuccessfulShards() > 0); + assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); + for (RemoteStoreStats stat: response.getRemoteStoreStats()) { + ShardRouting routing = stat.getShardRouting(); + RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); + if (routing.primary()) { + validateUploadStats(stats); + } else { + validateDownloadStats(stats); + } + } } } @@ -127,7 +206,7 @@ private void indexDocs() { } } - private void assertResponseStats(RemoteRefreshSegmentTracker.Stats stats) { + private void validateUploadStats(RemoteRefreshSegmentTracker.Stats stats) { assertEquals(0, stats.refreshTimeLagMs); assertEquals(stats.localRefreshNumber, stats.remoteRefreshNumber); assertTrue(stats.uploadBytesStarted > 0); @@ -144,6 +223,20 @@ private void assertResponseStats(RemoteRefreshSegmentTracker.Stats stats) { assertTrue(stats.uploadTimeMovingAverage > 0); } + private void validateDownloadStats(RemoteRefreshSegmentTracker.Stats stats) { + assertTrue(stats.lastDownloadTimestampMs > 0); + assertTrue(stats.totalDownloadsStarted > 0); + assertTrue(stats.totalDownloadsSucceeded > 0); + assertEquals(stats.totalDownloadsFailed, 0); + assertTrue(stats.downloadBytesStarted > 0); + assertTrue(stats.downloadBytesSucceeded > 0); + assertEquals(stats.downloadBytesFailed, 0); + assertTrue(stats.lastSuccessfulSegmentDownloadBytes > 0); + assertTrue(stats.downloadBytesMovingAverage > 0); + assertTrue(stats.downloadBytesPerSecMovingAverage > 0); + assertTrue(stats.downloadTimeMovingAverage > 0); + } + private IndexResponse indexSingleDoc() { return client().prepareIndex(INDEX_NAME) .setId(UUIDs.randomBase64UUID()) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index 117ad8ac2bf59..86a0c53732f69 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -8,9 +8,14 @@ package org.opensearch.action.admin.cluster.remotestore.stats; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.logging.Loggers; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; @@ -24,70 +29,133 @@ */ public class RemoteStoreStats implements Writeable, ToXContentFragment { - private final RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats; + private final RemoteRefreshSegmentTracker.Stats remoteSegmentShardStats; - public RemoteStoreStats(RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats) { - this.remoteSegmentUploadShardStats = remoteSegmentUploadShardStats; + private final ShardRouting currentRouting; + + private final Logger logger; + + public RemoteStoreStats(RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats, ShardRouting currentRouting) { + this.remoteSegmentShardStats = remoteSegmentUploadShardStats; + this.currentRouting = currentRouting; + this.logger = Loggers.getLogger(getClass(), this.currentRouting.getIndexName()); } public RemoteStoreStats(StreamInput in) throws IOException { - remoteSegmentUploadShardStats = in.readOptionalWriteable(RemoteRefreshSegmentTracker.Stats::new); + this.remoteSegmentShardStats = in.readOptionalWriteable(RemoteRefreshSegmentTracker.Stats::new); + this.currentRouting = new ShardRouting(in); + this.logger = Loggers.getLogger(getClass(), this.currentRouting.getIndexName()); } public RemoteRefreshSegmentTracker.Stats getStats() { - return remoteSegmentUploadShardStats; + return remoteSegmentShardStats; + } + + public ShardRouting getShardRouting() { + return currentRouting; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject() - .field(Fields.SHARD_ID, remoteSegmentUploadShardStats.shardId) - .field(Fields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentUploadShardStats.localRefreshClockTimeMs) - .field(Fields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentUploadShardStats.remoteRefreshClockTimeMs) - .field(Fields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentUploadShardStats.refreshTimeLagMs) - .field(Fields.REFRESH_LAG, remoteSegmentUploadShardStats.localRefreshNumber - remoteSegmentUploadShardStats.remoteRefreshNumber) - .field(Fields.BYTES_LAG, remoteSegmentUploadShardStats.bytesLag) - - .field(Fields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentUploadShardStats.rejectionCount) - .field(Fields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentUploadShardStats.consecutiveFailuresCount); - - builder.startObject(Fields.TOTAL_REMOTE_REFRESH); - builder.field(SubFields.STARTED, remoteSegmentUploadShardStats.totalUploadsStarted) - .field(SubFields.SUCCEEDED, remoteSegmentUploadShardStats.totalUploadsSucceeded) - .field(SubFields.FAILED, remoteSegmentUploadShardStats.totalUploadsFailed); - builder.endObject(); - - builder.startObject(Fields.TOTAL_UPLOADS_IN_BYTES); - builder.field(SubFields.STARTED, remoteSegmentUploadShardStats.uploadBytesStarted) - .field(SubFields.SUCCEEDED, remoteSegmentUploadShardStats.uploadBytesSucceeded) - .field(SubFields.FAILED, remoteSegmentUploadShardStats.uploadBytesFailed); - builder.endObject(); - - builder.startObject(Fields.REMOTE_REFRESH_SIZE_IN_BYTES); - builder.field(SubFields.LAST_SUCCESSFUL, remoteSegmentUploadShardStats.lastSuccessfulRemoteRefreshBytes); - builder.field(SubFields.MOVING_AVG, remoteSegmentUploadShardStats.uploadBytesMovingAverage); - builder.endObject(); - - builder.startObject(Fields.UPLOAD_LATENCY_IN_BYTES_PER_SEC); - builder.field(SubFields.MOVING_AVG, remoteSegmentUploadShardStats.uploadBytesPerSecMovingAverage); - builder.endObject(); - builder.startObject(Fields.REMOTE_REFRESH_LATENCY_IN_MILLIS); - builder.field(SubFields.MOVING_AVG, remoteSegmentUploadShardStats.uploadTimeMovingAverage); + builder.startObject(); + builder.startObject(Fields.ROUTING); + builder.field(RoutingFields.STATE, currentRouting.state()); + builder.field(RoutingFields.PRIMARY, currentRouting.primary()); + builder.field(RoutingFields.NODE_ID, currentRouting.currentNodeId()); builder.endObject(); + builder.startObject(Fields.SEGMENT); + if (!currentRouting.primary()) { + builder.startObject(SubFields.DOWNLOAD); + builder.field(DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); + builder.startObject(DownloadStatsFields.TOTAL_FILE_DOWNLOADS) + .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); + builder.endObject(); + builder.startObject(DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES) + .field(SubFields.STARTED, remoteSegmentShardStats.downloadBytesStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.downloadBytesSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.downloadBytesFailed); + builder.endObject(); + builder.startObject(DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES) + .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulSegmentDownloadBytes) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesMovingAverage); + builder.endObject(); + builder.startObject(DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesPerSecMovingAverage); + builder.endObject(); + builder.startObject(DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadTimeMovingAverage); + builder.endObject(); + builder.endObject(); + } else { + builder.startObject(SubFields.DOWNLOAD); + builder.endObject(); + } + if (currentRouting.primary()) { + builder.startObject(SubFields.UPLOAD) + .field(UploadFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) + .field(UploadFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) + .field(UploadFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) + .field(UploadFields.REFRESH_LAG, remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber) + .field(UploadFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) + .field(UploadFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) + .field(UploadFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); + builder.startObject(UploadFields.TOTAL_REMOTE_REFRESH) + .field(SubFields.STARTED, remoteSegmentShardStats.totalUploadsStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalUploadsSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.totalUploadsFailed); + builder.endObject(); + builder.startObject(UploadFields.TOTAL_UPLOADS_IN_BYTES) + .field(SubFields.STARTED, remoteSegmentShardStats.uploadBytesStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.uploadBytesSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.uploadBytesFailed); + builder.endObject(); + builder.startObject(UploadFields.REMOTE_REFRESH_SIZE_IN_BYTES) + .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulRemoteRefreshBytes) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesMovingAverage); + builder.endObject(); + builder.startObject(UploadFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesPerSecMovingAverage); + builder.endObject(); + builder.startObject(UploadFields.REMOTE_REFRESH_LATENCY_IN_MILLIS) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadTimeMovingAverage); + builder.endObject(); + builder.endObject(); + } else { + builder.startObject(SubFields.UPLOAD); + builder.endObject(); + } builder.endObject(); + return builder.endObject(); + } - return builder; + @Override + public String toString() { + return Strings.toString(XContentType.JSON, this, true, false); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalWriteable(remoteSegmentUploadShardStats); + out.writeOptionalWriteable(remoteSegmentShardStats); + currentRouting.writeTo(out); + } + + static final class Fields { + static final String ROUTING = "routing"; + static final String SEGMENT = "segment"; + static final String TRANSLOG = "translog"; } + static final class RoutingFields { + static final String STATE = "state"; + static final String PRIMARY = "primary"; + static final String NODE_ID = "node"; + } /** * Fields for remote store stats response */ - static final class Fields { + static final class UploadFields { static final String SHARD_ID = "shard_id"; /** @@ -151,6 +219,15 @@ static final class Fields { static final String REMOTE_REFRESH_LATENCY_IN_MILLIS = "remote_refresh_latency_in_millis"; } + static final class DownloadStatsFields { + static final String LAST_DOWNLOAD_TIMESTAMP = "last_download_timestamp"; + static final String TOTAL_FILE_DOWNLOADS = "total_file_downloads"; + static final String TOTAL_FILE_DOWNLOADS_IN_BYTES = "total_file_downloads_in_bytes"; + static final String DOWNLOAD_SIZE_IN_BYTES = "download_size_in_bytes"; + static final String DOWNLOAD_SPEED_IN_BYTES_PER_SEC = "download_speed_in_bytes_per_sec"; + static final String DOWNLOAD_LATENCY_IN_MILLIS = "download_latency_in_millis"; + } + /** * Reusable sub fields for {@link Fields} */ @@ -159,6 +236,9 @@ static final class SubFields { static final String SUCCEEDED = "succeeded"; static final String FAILED = "failed"; + static final String DOWNLOAD = "download"; + static final String UPLOAD = "upload"; + /** * Moving avg over last N values stat for a {@link Fields} */ diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java index 5a49f90b42b07..b81258d9bdd41 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java @@ -8,16 +8,24 @@ package org.opensearch.action.admin.cluster.remotestore.stats; +import org.apache.logging.log4j.Logger; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.logging.Loggers; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; /** * Remote Store stats response @@ -26,11 +34,15 @@ */ public class RemoteStoreStatsResponse extends BroadcastResponse { - private final RemoteStoreStats[] shards; + private final RemoteStoreStats[] remoteStoreStats; + private Map>> indexWiseStats; + + private final Logger logger; public RemoteStoreStatsResponse(StreamInput in) throws IOException { super(in); - shards = in.readArray(RemoteStoreStats::new, RemoteStoreStats[]::new); + remoteStoreStats = in.readArray(RemoteStoreStats::new, RemoteStoreStats[]::new); + this.logger = Loggers.getLogger(getClass(), "sample-index"); } public RemoteStoreStatsResponse( @@ -41,30 +53,86 @@ public RemoteStoreStatsResponse( List shardFailures ) { super(totalShards, successfulShards, failedShards, shardFailures); - this.shards = shards; + this.remoteStoreStats = shards; + this.logger = Loggers.getLogger(getClass(), "sample-index"); } - public RemoteStoreStats[] getShards() { - return this.shards; + public RemoteStoreStats[] getRemoteStoreStats() { + return this.remoteStoreStats; } public RemoteStoreStats getAt(int position) { - return shards[position]; + return remoteStoreStats[position]; + } + + public Map>> groupByIndexAndShards() { + if (indexWiseStats != null) { + return indexWiseStats; + } + Set indexNames = new HashSet<>(); + for (RemoteStoreStats shardStat : remoteStoreStats) { + String indexName = shardStat.getShardRouting().getIndexName(); + indexNames.add(indexName); + } + + Map>> indexWiseStats = new HashMap<>(); + for (String indexName : indexNames) { + Set perIndexStats = new HashSet<>(); + for (RemoteStoreStats shardStat : remoteStoreStats) { + if (shardStat.getShardRouting().getIndexName().equals(indexName)) { + perIndexStats.add(shardStat); + } + } + indexWiseStats.put(indexName, groupByShards(perIndexStats)); + } + this.indexWiseStats = indexWiseStats; + return indexWiseStats; + } + + public Map> groupByShards(Set perIndexStats) { + Map> shardStats = new HashMap<>(); + Set shardIds = new HashSet<>(); + for(RemoteStoreStats eachShardStats: perIndexStats) { + int shardId = eachShardStats.getShardRouting().getId(); + shardIds.add(shardId); + } + + for (Integer shardId: shardIds) { + List stats = new ArrayList<>(); + for (RemoteStoreStats perShardStats: perIndexStats) { + if (perShardStats.getShardRouting().getId() == shardId) { + stats.add(perShardStats); + } + } + shardStats.put(shardId, stats); + } + return shardStats; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeArray(shards); + out.writeArray(remoteStoreStats); } @Override protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { - builder.startArray("stats"); - for (RemoteStoreStats shard : shards) { - shard.toXContent(builder, params); + groupByIndexAndShards(); + builder.startObject("indices"); + for (String indexName: indexWiseStats.keySet()) { + builder.startObject(indexName); + builder.startObject("shards"); + for (int shardId: indexWiseStats.get(indexName).keySet()) { + builder.startArray(Integer.toString(shardId)); + for (RemoteStoreStats shardStat: indexWiseStats.get(indexName).get(shardId)) { + shardStat.toXContent(builder, params); + } + builder.endArray(); + } + builder.endObject(); + builder.endObject(); } - builder.endArray(); + builder.endObject(); } @Override diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java index ff974e32678d6..ead43bf56f713 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java @@ -22,6 +22,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; @@ -49,6 +50,8 @@ public class TransportRemoteStoreStatsAction extends TransportBroadcastByNodeAct RemoteStoreStats> { private final IndicesService indicesService; + + private final ClusterService clusterService; private final RemoteRefreshSegmentPressureService remoteRefreshSegmentPressureService; @Inject @@ -69,6 +72,7 @@ public TransportRemoteStoreStatsAction( RemoteStoreStatsRequest::new, ThreadPool.Names.MANAGEMENT ); + this.clusterService = clusterService; this.indicesService = indicesService; this.remoteRefreshSegmentPressureService = remoteRefreshSegmentPressureService; } @@ -95,7 +99,6 @@ protected ShardsIterator shards(ClusterState clusterState, RemoteStoreStatsReque || (shardRouting.currentNodeId() == null || shardRouting.currentNodeId().equals(clusterState.getNodes().getLocalNodeId())) ) - .filter(ShardRouting::primary) .filter( shardRouting -> Boolean.parseBoolean( clusterState.getMetadata().index(shardRouting.index()).getSettings().get(IndexMetadata.SETTING_REMOTE_STORE_ENABLED) @@ -157,7 +160,6 @@ protected RemoteStoreStats shardOperation(RemoteStoreStatsRequest request, Shard indexShard.shardId() ); assert Objects.nonNull(remoteRefreshSegmentTracker); - - return new RemoteStoreStats(remoteRefreshSegmentTracker.stats()); + return new RemoteStoreStats(remoteRefreshSegmentTracker.stats(), indexShard.routingEntry()); } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index d70ea16cf5fdd..27f18ccb2ea73 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -652,6 +652,9 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteRefreshSegmentPressureSettings.UPLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, RemoteRefreshSegmentPressureSettings.UPLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, RemoteRefreshSegmentPressureSettings.UPLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, + RemoteRefreshSegmentPressureSettings.DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, + RemoteRefreshSegmentPressureSettings.DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, + RemoteRefreshSegmentPressureSettings.DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, // Related to monitoring of task cancellation TaskCancellationMonitoringSettings.IS_ENABLED_SETTING, diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java index 280381a7b6109..dc0c153f72e2d 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java @@ -78,6 +78,9 @@ public void afterIndexShardCreated(IndexShard indexShard) { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ) ); @@ -134,6 +137,18 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateUploadTimeMsMovingAverageWindowSize, updatedSize); } + void updateDownloadBytesMovingAverageWindowSize(int updatedSize) { + updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateDownloadBytesPerSecMovingAverageWindowSize, updatedSize); + } + + void updateDownloadBytesPerSecMovingAverageWindowSize(int updatedSize) { + updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateDownloadBytesMovingAverageWindowSize, updatedSize); + } + + void updateDownloadTimeMovingAverageWindowSize(int updatedSize) { + updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateDownloadTimeMsMovingAverageWindowSize, updatedSize); + } + void updateMovingAverageWindowSize(BiConsumer biConsumer, int updatedSize) { trackerMap.values().forEach(tracker -> biConsumer.accept(tracker, updatedSize)); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java index 2a098b8f7a89b..2506d1a683ea0 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java @@ -29,6 +29,9 @@ private static class Defaults { private static final int UPLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE = 20; private static final int UPLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE = 20; private static final int UPLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE = 20; + private static final int DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE = 20; + private static final int DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE = 20; + private static final int DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE = 20; private static final int MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE = 5; } @@ -87,6 +90,30 @@ private static class Defaults { Setting.Property.NodeScope ); + public static final Setting DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE = Setting.intSetting( + "remote_store.segment.pressure.download_size_in_bytes_moving_average_window_size", + Defaults.DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, + Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final Setting DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE = Setting.intSetting( + "remote_store.segment.pressure.download_speed_in_bytes_per_sec_moving_average_window_size", + Defaults.DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, + Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final Setting DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE = Setting.intSetting( + "remote_store.segment.pressure.download_time_moving_average_window_size", + Defaults.DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, + Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + private volatile boolean remoteRefreshSegmentPressureEnabled; private volatile long minRefreshSeqNoLagLimit; @@ -103,6 +130,12 @@ private static class Defaults { private volatile int uploadTimeMovingAverageWindowSize; + private volatile int downloadBytesMovingAverageWindowSize; + + private volatile int downloadBytesPerSecMovingAverageWindowSize; + + private volatile int downloadTimeMovingAverageWindowSize; + public RemoteRefreshSegmentPressureSettings( ClusterService clusterService, Settings settings, @@ -145,6 +178,27 @@ public RemoteRefreshSegmentPressureSettings( remoteUploadPressureService::updateUploadTimeMsMovingAverageWindowSize ); clusterSettings.addSettingsUpdateConsumer(UPLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, this::setUploadTimeMovingAverageWindowSize); + + this.downloadBytesMovingAverageWindowSize = DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE.get(settings); + clusterSettings.addSettingsUpdateConsumer( + DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, + remoteUploadPressureService::updateDownloadBytesMovingAverageWindowSize + ); + clusterSettings.addSettingsUpdateConsumer(DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, this::setDownloadBytesMovingAverageWindowSize); + + this.downloadBytesPerSecMovingAverageWindowSize = DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE.get(settings); + clusterSettings.addSettingsUpdateConsumer( + DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, + remoteUploadPressureService::updateDownloadBytesPerSecMovingAverageWindowSize + ); + clusterSettings.addSettingsUpdateConsumer(DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, this::setDownloadBytesPerSecMovingAverageWindowSize); + + this.downloadTimeMovingAverageWindowSize = DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE.get(settings); + clusterSettings.addSettingsUpdateConsumer( + DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, + remoteUploadPressureService::updateDownloadTimeMovingAverageWindowSize + ); + clusterSettings.addSettingsUpdateConsumer(DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, this::setDownloadTimeMovingAverageWindowSize); } public boolean isRemoteRefreshSegmentPressureEnabled() { @@ -210,4 +264,28 @@ public int getUploadTimeMovingAverageWindowSize() { public void setUploadTimeMovingAverageWindowSize(int uploadTimeMovingAverageWindowSize) { this.uploadTimeMovingAverageWindowSize = uploadTimeMovingAverageWindowSize; } + + public int getDownloadBytesMovingAverageWindowSize() { + return downloadBytesMovingAverageWindowSize; + } + + public void setDownloadBytesMovingAverageWindowSize(int downloadBytesMovingAverageWindowSize) { + this.downloadBytesMovingAverageWindowSize = downloadBytesMovingAverageWindowSize; + } + + public int getDownloadBytesPerSecMovingAverageWindowSize() { + return downloadBytesPerSecMovingAverageWindowSize; + } + + public void setDownloadBytesPerSecMovingAverageWindowSize(int downloadBytesPerSecMovingAverageWindowSize) { + this.downloadBytesPerSecMovingAverageWindowSize = downloadBytesPerSecMovingAverageWindowSize; + } + + public int getDownloadTimeMovingAverageWindowSize() { + return downloadBytesPerSecMovingAverageWindowSize; + } + + public void setDownloadTimeMovingAverageWindowSize(int downloadBytesPerSecMovingAverageWindowSize) { + this.downloadBytesPerSecMovingAverageWindowSize = downloadBytesPerSecMovingAverageWindowSize; + } } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index 919d34258a507..4d849bff30e63 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -52,6 +52,8 @@ public class RemoteRefreshSegmentTracker { */ private volatile long localRefreshClockTimeMs; + private volatile long lastDownloadTimestampMs; + /** * Sequence number of the most recent remote refresh. */ @@ -82,6 +84,8 @@ public class RemoteRefreshSegmentTracker { */ private volatile long lastSuccessfulRemoteRefreshBytes; + private volatile long lastSuccessfulSegmentDownloadBytes; + /** * Cumulative sum of size in bytes of segment files for which upload has started during remote refresh. */ @@ -97,6 +101,10 @@ public class RemoteRefreshSegmentTracker { */ private volatile long uploadBytesSucceeded; + private volatile long downloadBytesStarted; + private volatile long downloadBytesFailed; + private volatile long downloadBytesSucceeded; + /** * Cumulative sum of count of remote refreshes that have started. */ @@ -112,6 +120,10 @@ public class RemoteRefreshSegmentTracker { */ private volatile long totalUploadsSucceeded; + private volatile long totalDownloadsStarted; + private volatile long totalDownloadsSucceeded; + private volatile long totalDownloadsFailed; + /** * Cumulative sum of rejection counts for this shard. */ @@ -153,6 +165,10 @@ public class RemoteRefreshSegmentTracker { */ private final Object uploadBytesMutex = new Object(); + private final AtomicReference downloadBytesMovingAverageReference; + + private final Object downloadBytesMutex = new Object(); + /** * Provides moving average over the last N upload speed (in bytes/s) of segment files uploaded as part of remote refresh. * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. @@ -161,6 +177,10 @@ public class RemoteRefreshSegmentTracker { private final Object uploadBytesPerSecMutex = new Object(); + private final AtomicReference downloadBytesPerSecMovingAverageReference; + + private final Object downloadBytesPerSecMutex = new Object(); + /** * Provides moving average over the last N overall upload time (in nanos) as part of remote refresh.N is window size. * Wrapped with {@code AtomicReference} for dynamic changes in window size. @@ -169,11 +189,18 @@ public class RemoteRefreshSegmentTracker { private final Object uploadTimeMsMutex = new Object(); + private final AtomicReference downloadTimeMovingAverageReference; + + private final Object downloadTimeMutex = new Object(); + public RemoteRefreshSegmentTracker( ShardId shardId, int uploadBytesMovingAverageWindowSize, int uploadBytesPerSecMovingAverageWindowSize, - int uploadTimeMsMovingAverageWindowSize + int uploadTimeMsMovingAverageWindowSize, + int downloadBytesMovingAverageWindowSize, + int downloadBytesPerSecMovingAverageWindowSize, + int downloadTimeMovingAverageWindowSize ) { this.shardId = shardId; // Both the local refresh time and remote refresh time are set with current time to give consistent view of time lag when it arises. @@ -186,7 +213,9 @@ public RemoteRefreshSegmentTracker( uploadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesMovingAverageWindowSize)); uploadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesPerSecMovingAverageWindowSize)); uploadTimeMsMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadTimeMsMovingAverageWindowSize)); - + downloadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(downloadBytesMovingAverageWindowSize)); + downloadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(downloadBytesPerSecMovingAverageWindowSize)); + downloadTimeMovingAverageReference = new AtomicReference<>(new MovingAverage(downloadTimeMovingAverageWindowSize)); latestLocalFileNameLengthMap = new HashMap<>(); } @@ -216,6 +245,10 @@ public long getLocalRefreshClockTimeMs() { return localRefreshClockTimeMs; } + public long getLastDownloadTimestampMs() { + return lastDownloadTimestampMs; + } + public void updateLocalRefreshTimeMs(long localRefreshTimeMs) { assert localRefreshTimeMs >= this.localRefreshTimeMs : "newLocalRefreshTimeMs=" + localRefreshTimeMs @@ -230,6 +263,10 @@ public void updateLocalRefreshClockTimeMs(long localRefreshClockTimeMs) { this.localRefreshClockTimeMs = localRefreshClockTimeMs; } + public void updateLastDownloadTimestampMs(long downloadTimestampInMs) { + this.lastDownloadTimestampMs = downloadTimestampInMs; + } + long getRemoteRefreshSeqNo() { return remoteRefreshSeqNo; } @@ -310,10 +347,38 @@ public void addUploadBytesSucceeded(long size) { uploadBytesSucceeded += size; } + public long getDownloadBytesStarted() { + return downloadBytesStarted; + } + + public void addDownloadBytesStarted(long size) { + downloadBytesStarted += size; + } + + public long getDownloadBytesFailed() { + return downloadBytesFailed; + } + + public void addDownloadBytesFailed(long size) { + downloadBytesFailed += size; + } + + public long getDownloadBytesSucceeded() { + return downloadBytesSucceeded; + } + + public void addDownloadBytesSucceeded(long size) { + downloadBytesSucceeded += size; + } + public long getInflightUploadBytes() { return uploadBytesStarted - uploadBytesFailed - uploadBytesSucceeded; } + public long getInflightDownloadBytes() { + return downloadBytesStarted - downloadBytesFailed - downloadBytesSucceeded; + } + public long getTotalUploadsStarted() { return totalUploadsStarted; } @@ -344,6 +409,34 @@ public long getInflightUploads() { return totalUploadsStarted - totalUploadsFailed - totalUploadsSucceeded; } + public long getTotalDownloadsStarted() { + return totalDownloadsStarted; + } + + public void incrementTotalDownloadsStarted() { + totalDownloadsStarted += 1; + } + + public long getTotalDownloadsFailed() { + return totalDownloadsFailed; + } + + public void incrementTotalDownloadsFailed() { + totalDownloadsFailed += 1; + } + + public long getTotalDownloadsSucceeded() { + return totalDownloadsSucceeded; + } + + public void incrementTotalDownloadsSucceeded() { + totalDownloadsSucceeded += 1; + } + + public long getInflightDownloads() { + return totalDownloadsStarted - totalDownloadsFailed - totalDownloadsSucceeded; + } + public long getRejectionCount() { return rejectionCount.get(); } @@ -422,6 +515,32 @@ void updateUploadBytesMovingAverageWindowSize(int updatedSize) { } } + boolean isDownloadBytesAverageReady() { + return downloadBytesMovingAverageReference.get().isReady(); + } + + double getDownloadBytesAverage() { + return uploadBytesMovingAverageReference.get().getAverage(); + } + + public void addDownloadBytes(long size) { + lastSuccessfulSegmentDownloadBytes = size; + synchronized (downloadBytesMutex) { + this.downloadBytesMovingAverageReference.get().record(size); + } + } + + /** + * Updates the window size for data collection of upload bytes. This also resets any data collected so far. + * + * @param updatedSize the updated size + */ + void updateDownloadBytesMovingAverageWindowSize(int updatedSize) { + synchronized (downloadBytesMutex) { + this.downloadBytesMovingAverageReference.set(this.downloadBytesMovingAverageReference.get().copyWithSize(updatedSize)); + } + } + boolean isUploadBytesPerSecAverageReady() { return uploadBytesPerSecMovingAverageReference.get().isReady(); } @@ -436,6 +555,31 @@ public void addUploadBytesPerSec(long bytesPerSec) { } } + /** + * Updates the window size for data collection of upload bytes per second. This also resets any data collected so far. + * + * @param updatedSize the updated size + */ + void updateDownloadBytesPerSecMovingAverageWindowSize(int updatedSize) { + synchronized (downloadBytesPerSecMutex) { + this.downloadBytesPerSecMovingAverageReference.set(this.downloadBytesPerSecMovingAverageReference.get().copyWithSize(updatedSize)); + } + } + + boolean isDownloadBytesPerSecAverageReady() { + return uploadBytesPerSecMovingAverageReference.get().isReady(); + } + + double getDownloadBytesPerSecAverage() { + return uploadBytesPerSecMovingAverageReference.get().getAverage(); + } + + public void addDownloadBytesPerSec(long bytesPerSec) { + synchronized (downloadBytesPerSecMutex) { + this.downloadBytesPerSecMovingAverageReference.get().record(bytesPerSec); + } + } + /** * Updates the window size for data collection of upload bytes per second. This also resets any data collected so far. * @@ -472,26 +616,62 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { } } + boolean isDownloadTimeAverageReady() { + return downloadTimeMovingAverageReference.get().isReady(); + } + + double getDownloadTimeAverage() { + return downloadTimeMovingAverageReference.get().getAverage(); + } + + public void addDownloadTime(long timeMs) { + synchronized (downloadTimeMutex) { + this.downloadTimeMovingAverageReference.get().record(timeMs); + } + } + + /** + * Updates the window size for data collection of upload time (ms). This also resets any data collected so far. + * + * @param updatedSize the updated size + */ + void updateDownloadTimeMsMovingAverageWindowSize(int updatedSize) { + synchronized (downloadTimeMutex) { + this.downloadTimeMovingAverageReference.set(this.downloadTimeMovingAverageReference.get().copyWithSize(updatedSize)); + } + } + public RemoteRefreshSegmentTracker.Stats stats() { return new RemoteRefreshSegmentTracker.Stats( shardId, localRefreshClockTimeMs, remoteRefreshClockTimeMs, timeMsLag, + lastDownloadTimestampMs, localRefreshSeqNo, remoteRefreshSeqNo, uploadBytesStarted, uploadBytesSucceeded, uploadBytesFailed, + downloadBytesStarted, + downloadBytesSucceeded, + downloadBytesFailed, totalUploadsStarted, totalUploadsSucceeded, totalUploadsFailed, + totalDownloadsStarted, + totalDownloadsSucceeded, + totalDownloadsFailed, rejectionCount.get(), failures.length(), lastSuccessfulRemoteRefreshBytes, uploadBytesMovingAverageReference.get().getAverage(), uploadBytesPerSecMovingAverageReference.get().getAverage(), uploadTimeMsMovingAverageReference.get().getAverage(), + lastSuccessfulSegmentDownloadBytes, + downloadBytesMovingAverageReference.get().getAverage(), + downloadBytesPerSecMovingAverageReference.get().getAverage(), + downloadTimeMovingAverageReference.get().getAverage(), getBytesLag() ); } @@ -507,20 +687,31 @@ public static class Stats implements Writeable { public final long localRefreshClockTimeMs; public final long remoteRefreshClockTimeMs; public final long refreshTimeLagMs; + public final long lastDownloadTimestampMs; public final long localRefreshNumber; public final long remoteRefreshNumber; public final long uploadBytesStarted; public final long uploadBytesFailed; public final long uploadBytesSucceeded; + public final long downloadBytesStarted; + public final long downloadBytesFailed; + public final long downloadBytesSucceeded; public final long totalUploadsStarted; public final long totalUploadsFailed; public final long totalUploadsSucceeded; + public final long totalDownloadsStarted; + public final long totalDownloadsFailed; + public final long totalDownloadsSucceeded; public final long rejectionCount; public final long consecutiveFailuresCount; public final long lastSuccessfulRemoteRefreshBytes; public final double uploadBytesMovingAverage; public final double uploadBytesPerSecMovingAverage; public final double uploadTimeMovingAverage; + public final long lastSuccessfulSegmentDownloadBytes; + public final double downloadBytesMovingAverage; + public final double downloadBytesPerSecMovingAverage; + public final double downloadTimeMovingAverage; public final long bytesLag; public Stats( @@ -528,40 +719,62 @@ public Stats( long localRefreshClockTimeMs, long remoteRefreshClockTimeMs, long refreshTimeLagMs, + long lastDownloadTimestampMs, long localRefreshNumber, long remoteRefreshNumber, long uploadBytesStarted, long uploadBytesSucceeded, long uploadBytesFailed, + long downloadBytesStarted, + long downloadBytesSucceeded, + long downloadBytesFailed, long totalUploadsStarted, long totalUploadsSucceeded, long totalUploadsFailed, + long totalDownloadsStarted, + long totalDownloadsSucceeded, + long totalDownloadsFailed, long rejectionCount, long consecutiveFailuresCount, long lastSuccessfulRemoteRefreshBytes, double uploadBytesMovingAverage, double uploadBytesPerSecMovingAverage, double uploadTimeMovingAverage, + long lastSuccessfulSegmentDownloadBytes, + double downloadBytesMovingAverage, + double downloadBytesPerSecMovingAverage, + double downloadTimeMovingAverage, long bytesLag ) { this.shardId = shardId; this.localRefreshClockTimeMs = localRefreshClockTimeMs; this.remoteRefreshClockTimeMs = remoteRefreshClockTimeMs; this.refreshTimeLagMs = refreshTimeLagMs; + this.lastDownloadTimestampMs = lastDownloadTimestampMs; this.localRefreshNumber = localRefreshNumber; this.remoteRefreshNumber = remoteRefreshNumber; this.uploadBytesStarted = uploadBytesStarted; this.uploadBytesFailed = uploadBytesFailed; this.uploadBytesSucceeded = uploadBytesSucceeded; + this.downloadBytesStarted = downloadBytesStarted; + this.downloadBytesFailed = downloadBytesFailed; + this.downloadBytesSucceeded = downloadBytesSucceeded; this.totalUploadsStarted = totalUploadsStarted; this.totalUploadsFailed = totalUploadsFailed; this.totalUploadsSucceeded = totalUploadsSucceeded; + this.totalDownloadsStarted = totalDownloadsStarted; + this.totalDownloadsFailed = totalDownloadsFailed; + this.totalDownloadsSucceeded = totalDownloadsSucceeded; this.rejectionCount = rejectionCount; this.consecutiveFailuresCount = consecutiveFailuresCount; this.lastSuccessfulRemoteRefreshBytes = lastSuccessfulRemoteRefreshBytes; this.uploadBytesMovingAverage = uploadBytesMovingAverage; this.uploadBytesPerSecMovingAverage = uploadBytesPerSecMovingAverage; this.uploadTimeMovingAverage = uploadTimeMovingAverage; + this.lastSuccessfulSegmentDownloadBytes = lastSuccessfulSegmentDownloadBytes; + this.downloadBytesMovingAverage = downloadBytesMovingAverage; + this.downloadBytesPerSecMovingAverage = downloadBytesPerSecMovingAverage; + this.downloadTimeMovingAverage = downloadTimeMovingAverage; this.bytesLag = bytesLag; } @@ -571,20 +784,31 @@ public Stats(StreamInput in) throws IOException { this.localRefreshClockTimeMs = in.readLong(); this.remoteRefreshClockTimeMs = in.readLong(); this.refreshTimeLagMs = in.readLong(); + this.lastDownloadTimestampMs = in.readLong(); this.localRefreshNumber = in.readLong(); this.remoteRefreshNumber = in.readLong(); this.uploadBytesStarted = in.readLong(); this.uploadBytesFailed = in.readLong(); this.uploadBytesSucceeded = in.readLong(); + this.downloadBytesStarted = in.readLong(); + this.downloadBytesFailed = in.readLong(); + this.downloadBytesSucceeded = in.readLong(); this.totalUploadsStarted = in.readLong(); this.totalUploadsFailed = in.readLong(); this.totalUploadsSucceeded = in.readLong(); + this.totalDownloadsStarted = in.readLong(); + this.totalDownloadsFailed = in.readLong(); + this.totalDownloadsSucceeded = in.readLong(); this.rejectionCount = in.readLong(); this.consecutiveFailuresCount = in.readLong(); this.lastSuccessfulRemoteRefreshBytes = in.readLong(); this.uploadBytesMovingAverage = in.readDouble(); this.uploadBytesPerSecMovingAverage = in.readDouble(); this.uploadTimeMovingAverage = in.readDouble(); + this.lastSuccessfulSegmentDownloadBytes = in.readLong(); + this.downloadBytesMovingAverage = in.readDouble(); + this.downloadBytesPerSecMovingAverage = in.readDouble(); + this.downloadTimeMovingAverage = in.readDouble(); this.bytesLag = in.readLong(); } catch (IOException e) { throw e; @@ -597,20 +821,31 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(localRefreshClockTimeMs); out.writeLong(remoteRefreshClockTimeMs); out.writeLong(refreshTimeLagMs); + out.writeLong(lastDownloadTimestampMs); out.writeLong(localRefreshNumber); out.writeLong(remoteRefreshNumber); out.writeLong(uploadBytesStarted); out.writeLong(uploadBytesFailed); out.writeLong(uploadBytesSucceeded); + out.writeLong(downloadBytesStarted); + out.writeLong(downloadBytesFailed); + out.writeLong(downloadBytesSucceeded); out.writeLong(totalUploadsStarted); out.writeLong(totalUploadsFailed); out.writeLong(totalUploadsSucceeded); + out.writeLong(totalDownloadsStarted); + out.writeLong(totalDownloadsFailed); + out.writeLong(totalDownloadsSucceeded); out.writeLong(rejectionCount); out.writeLong(consecutiveFailuresCount); out.writeLong(lastSuccessfulRemoteRefreshBytes); out.writeDouble(uploadBytesMovingAverage); out.writeDouble(uploadBytesPerSecMovingAverage); out.writeDouble(uploadTimeMovingAverage); + out.writeLong(lastSuccessfulSegmentDownloadBytes); + out.writeDouble(downloadBytesMovingAverage); + out.writeDouble(downloadBytesPerSecMovingAverage); + out.writeDouble(downloadTimeMovingAverage); out.writeLong(bytesLag); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 01c0a12d463ea..a974158510cc7 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -147,6 +147,7 @@ import org.opensearch.index.recovery.RecoveryStats; import org.opensearch.index.refresh.RefreshStats; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; +import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.search.stats.ShardSearchStats; import org.opensearch.index.seqno.ReplicationTracker; @@ -4628,7 +4629,13 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re storeDirectory = store.directory(); } Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); - copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal); + copySegmentFiles( + storeDirectory, + remoteDirectory, + null, + uploadedSegments, + remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), + overrideLocal); if (refreshLevelSegmentSync && remoteSegmentMetadata != null) { try ( @@ -4712,6 +4719,7 @@ public void syncSegmentsFromGivenRemoteSegmentStore( sourceRemoteDirectory, remoteDirectory, uploadedSegments, + remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), overrideLocal ); if (segmentsNFile != null) { @@ -4745,6 +4753,7 @@ private String copySegmentFiles( RemoteSegmentStoreDirectory sourceRemoteDirectory, RemoteSegmentStoreDirectory targetRemoteDirectory, Map uploadedSegments, + RemoteRefreshSegmentTracker downloadStatsTracker, boolean overrideLocal ) throws IOException { List downloadedSegments = new ArrayList<>(); @@ -4758,11 +4767,21 @@ private String copySegmentFiles( } } for (String file : uploadedSegments.keySet()) { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata segmentMetadata = uploadedSegments.get(file); long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - storeDirectory.sync(Collections.singleton(file)); - downloadedSegments.add(file); + long startTimeInMs = System.currentTimeMillis(); + long segmentSizeInBytes = segmentMetadata.getLength(); + beforeSegmentDownload(downloadStatsTracker, segmentSizeInBytes); + try { + storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); + storeDirectory.sync(Collections.singleton(file)); + downloadedSegments.add(file); + afterSegmentDownloadCompleted(downloadStatsTracker, segmentSizeInBytes, startTimeInMs); + } catch (IOException e) { + afterSegmentDownloadFailed(downloadStatsTracker, segmentSizeInBytes); + throw e; + } } else { skippedSegments.add(file); } @@ -4796,6 +4815,28 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo return false; } + private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTracker, long incomingFileSize) { + downloadStatsTracker.incrementTotalDownloadsStarted(); + downloadStatsTracker.addDownloadBytesStarted(incomingFileSize); + } + + private void afterSegmentDownloadCompleted(RemoteRefreshSegmentTracker downloadStatsTracker, long downloadedFileSize, long startTimeInNs) { + long currentTime = System.currentTimeMillis(); + downloadStatsTracker.updateLastDownloadTimestampMs(currentTime); + downloadStatsTracker.incrementTotalDownloadsSucceeded(); + downloadStatsTracker.addDownloadBytes(downloadedFileSize); + downloadStatsTracker.addDownloadBytesSucceeded(downloadedFileSize); + long timeTakenInMS = currentTime - startTimeInNs; + downloadStatsTracker.addDownloadTime(timeTakenInMS); + downloadStatsTracker.addDownloadBytesPerSec((downloadedFileSize * 1_000L) / timeTakenInMS); + + } + + private void afterSegmentDownloadFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { + downloadStatsTracker.incrementTotalDownloadsFailed(); + downloadStatsTracker.addDownloadBytesFailed(failedFileSize); + } + /** * Returns the maximum sequence number of either update or delete operations have been processed in this shard * or the sequence number from {@link #advanceMaxSeqNoOfUpdatesOrDeletes(long)}. An index request is considered diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java index 7d8e1ad5c7016..65baefe8c2c7d 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java @@ -9,6 +9,7 @@ package org.opensearch.action.admin.cluster.remotestore.stats; import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; @@ -24,6 +25,7 @@ import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.compareStatsResponse; import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createPressureTrackerStats; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createShardRouting; import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; public class RemoteStoreStatsResponseTests extends OpenSearchTestCase { @@ -45,9 +47,10 @@ public void tearDown() throws Exception { public void testSerialization() throws Exception { RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); - RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats); + ShardRouting primaryShardRouting = createShardRouting(shardId, true); + RemoteStoreStats primaryShardStats = new RemoteStoreStats(pressureTrackerStats, primaryShardRouting); RemoteStoreStatsResponse statsResponse = new RemoteStoreStatsResponse( - new RemoteStoreStats[] { stats }, + new RemoteStoreStats[] { primaryShardStats }, 1, 1, 0, @@ -58,15 +61,15 @@ public void testSerialization() throws Exception { statsResponse.toXContent(builder, EMPTY_PARAMS); Map jsonResponseObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()) .v2(); - - ArrayList> statsObjectArray = (ArrayList>) jsonResponseObject.get("stats"); - assertEquals(statsObjectArray.size(), 1); - Map statsObject = statsObjectArray.get(0); - Map shardsObject = (Map) jsonResponseObject.get("_shards"); - - assertEquals(shardsObject.get("total"), 1); - assertEquals(shardsObject.get("successful"), 1); - assertEquals(shardsObject.get("failed"), 0); - compareStatsResponse(statsObject, pressureTrackerStats); + Map metadataShardsObject = (Map) jsonResponseObject.get("_shards"); + assertEquals(metadataShardsObject.get("total"), 1); + assertEquals(metadataShardsObject.get("successful"), 1); + assertEquals(metadataShardsObject.get("failed"), 0); + Map indicesObject = (Map) jsonResponseObject.get("indices"); + assertTrue(indicesObject.containsKey("index")); + Map shardsObject = (Map)((Map) indicesObject.get("index")).get("shards"); + ArrayList> perShardNumberObject = (ArrayList>) shardsObject.get("0"); + assertEquals(perShardNumberObject.size(), 1); + compareStatsResponse(perShardNumberObject.get(0), pressureTrackerStats, primaryShardRouting); } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index 096dfc9145d57..47c2b4cc11683 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -8,80 +8,156 @@ package org.opensearch.action.admin.cluster.remotestore.stats; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.index.shard.ShardId; import java.util.Map; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.opensearch.test.OpenSearchTestCase.assertEquals; +import static org.opensearch.test.OpenSearchTestCase.randomAlphaOfLength; /** * Helper utilities for Remote Store stats tests */ public class RemoteStoreStatsTestHelper { static RemoteRefreshSegmentTracker.Stats createPressureTrackerStats(ShardId shardId) { - return new RemoteRefreshSegmentTracker.Stats(shardId, 101, 102, 100, 3, 2, 10, 5, 5, 10, 5, 5, 3, 2, 5, 2, 3, 4, 9); + return new RemoteRefreshSegmentTracker.Stats(shardId, + 101, 102, 100, + 3, 10, 2, + 10, 5, 5, + 10, 10, 5, + 5, 5, 5, + 5, 5, 5, + 3, 2, 5, + 2, 3, 4, + 9, 3, 4, + 9, 5); } - static void compareStatsResponse(Map statsObject, RemoteRefreshSegmentTracker.Stats pressureTrackerStats) { - assertEquals(statsObject.get(RemoteStoreStats.Fields.SHARD_ID), pressureTrackerStats.shardId.toString()); - assertEquals(statsObject.get(RemoteStoreStats.Fields.LOCAL_REFRESH_TIMESTAMP), (int) pressureTrackerStats.localRefreshClockTimeMs); - assertEquals( - statsObject.get(RemoteStoreStats.Fields.REMOTE_REFRESH_TIMESTAMP), - (int) pressureTrackerStats.remoteRefreshClockTimeMs - ); - assertEquals(statsObject.get(RemoteStoreStats.Fields.REFRESH_TIME_LAG_IN_MILLIS), (int) pressureTrackerStats.refreshTimeLagMs); - assertEquals( - statsObject.get(RemoteStoreStats.Fields.REFRESH_LAG), - (int) (pressureTrackerStats.localRefreshNumber - pressureTrackerStats.remoteRefreshNumber) - ); - assertEquals(statsObject.get(RemoteStoreStats.Fields.BYTES_LAG), (int) pressureTrackerStats.bytesLag); + static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { + return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); + } + + static void compareStatsResponse(Map statsObject, RemoteRefreshSegmentTracker.Stats pressureTrackerStats, + ShardRouting routing + ) { + assertEquals(((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.NODE_ID), routing.currentNodeId()); + assertEquals(((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.STATE), routing.state().toString()); + assertEquals(((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.PRIMARY), routing.primary()); + + Map segment = ((Map) statsObject.get(RemoteStoreStats.Fields.SEGMENT)); + Map segmentDownloads = ((Map) segment.get(RemoteStoreStats.SubFields.DOWNLOAD)); + Map segmentUploads = ((Map) segment.get(RemoteStoreStats.SubFields.UPLOAD)); - assertEquals(statsObject.get(RemoteStoreStats.Fields.BACKPRESSURE_REJECTION_COUNT), (int) pressureTrackerStats.rejectionCount); - assertEquals( - statsObject.get(RemoteStoreStats.Fields.CONSECUTIVE_FAILURE_COUNT), - (int) pressureTrackerStats.consecutiveFailuresCount - ); + if(!routing.primary()) { + assertEquals(segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP), + (int) pressureTrackerStats.lastDownloadTimestampMs + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get(RemoteStoreStats.SubFields.STARTED), + (int) pressureTrackerStats.totalDownloadsStarted + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get(RemoteStoreStats.SubFields.SUCCEEDED), + (int) pressureTrackerStats.totalDownloadsSucceeded + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get(RemoteStoreStats.SubFields.FAILED), + (int) pressureTrackerStats.totalDownloadsFailed + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), + (int) pressureTrackerStats.downloadBytesStarted + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), + (int) pressureTrackerStats.downloadBytesSucceeded + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), + (int) pressureTrackerStats.downloadBytesFailed + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), + (int) pressureTrackerStats.lastSuccessfulSegmentDownloadBytes + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), + pressureTrackerStats.downloadBytesMovingAverage + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), + pressureTrackerStats.downloadBytesPerSecMovingAverage + ); + assertEquals( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), + pressureTrackerStats.downloadTimeMovingAverage + ); + assertTrue(segmentUploads.isEmpty()); + } else { + assertTrue(segmentDownloads.isEmpty()); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.LOCAL_REFRESH_TIMESTAMP), + (int) pressureTrackerStats.localRefreshClockTimeMs + ); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_TIMESTAMP), + (int) pressureTrackerStats.remoteRefreshClockTimeMs + ); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.REFRESH_TIME_LAG_IN_MILLIS), + (int) pressureTrackerStats.refreshTimeLagMs + ); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.REFRESH_LAG), + (int) (pressureTrackerStats.localRefreshNumber - pressureTrackerStats.remoteRefreshNumber) + ); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.BYTES_LAG), (int) pressureTrackerStats.bytesLag); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), - (int) pressureTrackerStats.uploadBytesStarted - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), - (int) pressureTrackerStats.uploadBytesSucceeded - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), - (int) pressureTrackerStats.uploadBytesFailed - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), - pressureTrackerStats.uploadBytesMovingAverage - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), - (int) pressureTrackerStats.lastSuccessfulRemoteRefreshBytes - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), - pressureTrackerStats.uploadBytesPerSecMovingAverage - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), - (int) pressureTrackerStats.totalUploadsStarted - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.SUCCEEDED), - (int) pressureTrackerStats.totalUploadsSucceeded - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), - (int) pressureTrackerStats.totalUploadsFailed - ); - assertEquals( - ((Map) statsObject.get(RemoteStoreStats.Fields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), - pressureTrackerStats.uploadTimeMovingAverage - ); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.BACKPRESSURE_REJECTION_COUNT), + (int) pressureTrackerStats.rejectionCount + ); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.CONSECUTIVE_FAILURE_COUNT), + (int) pressureTrackerStats.consecutiveFailuresCount + ); + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), + (int) pressureTrackerStats.uploadBytesStarted + ); + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), + (int) pressureTrackerStats.uploadBytesSucceeded + ); + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), + (int) pressureTrackerStats.uploadBytesFailed + ); + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), + pressureTrackerStats.uploadBytesMovingAverage + ); + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), + (int) pressureTrackerStats.lastSuccessfulRemoteRefreshBytes + ); + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), + pressureTrackerStats.uploadBytesPerSecMovingAverage + ); + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), + (int) pressureTrackerStats.totalUploadsStarted + ); + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.SUCCEEDED), + (int) pressureTrackerStats.totalUploadsSucceeded + ); + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), + (int) pressureTrackerStats.totalUploadsFailed + ); + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), + pressureTrackerStats.uploadTimeMovingAverage + ); + } } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 94d2eae31c040..988b662fae040 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -8,6 +8,7 @@ package org.opensearch.action.admin.cluster.remotestore.stats; +import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.StreamInput; @@ -25,6 +26,7 @@ import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.compareStatsResponse; import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createPressureTrackerStats; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createShardRouting; import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; public class RemoteStoreStatsTests extends OpenSearchTestCase { @@ -44,43 +46,66 @@ public void tearDown() throws Exception { threadPool.shutdownNow(); } - public void testXContentBuilder() throws IOException { + public void testXContentBuilderWithPrimaryShard() throws IOException { RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); - RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats); + ShardRouting routing = createShardRouting(shardId, true); + RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats, routing); XContentBuilder builder = XContentFactory.jsonBuilder(); stats.toXContent(builder, EMPTY_PARAMS); Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); - compareStatsResponse(jsonObject, pressureTrackerStats); + compareStatsResponse(jsonObject, pressureTrackerStats, routing); + } + + public void testXContentBuilderWithReplicaShard() throws IOException { + RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); + ShardRouting routing = createShardRouting(shardId, false); + RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats, routing); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + stats.toXContent(builder, EMPTY_PARAMS); + Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + compareStatsResponse(jsonObject, pressureTrackerStats, routing); } public void testSerialization() throws Exception { RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); - RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats); + RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats, createShardRouting(shardId, true)); try (BytesStreamOutput out = new BytesStreamOutput()) { stats.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - RemoteStoreStats deserializedStats = new RemoteStoreStats(in); - assertEquals(deserializedStats.getStats().shardId.toString(), stats.getStats().shardId.toString()); - assertEquals(deserializedStats.getStats().refreshTimeLagMs, stats.getStats().refreshTimeLagMs); - assertEquals(deserializedStats.getStats().localRefreshNumber, stats.getStats().localRefreshNumber); - assertEquals(deserializedStats.getStats().remoteRefreshNumber, stats.getStats().remoteRefreshNumber); - assertEquals(deserializedStats.getStats().uploadBytesStarted, stats.getStats().uploadBytesStarted); - assertEquals(deserializedStats.getStats().uploadBytesSucceeded, stats.getStats().uploadBytesSucceeded); - assertEquals(deserializedStats.getStats().uploadBytesFailed, stats.getStats().uploadBytesFailed); - assertEquals(deserializedStats.getStats().totalUploadsStarted, stats.getStats().totalUploadsStarted); - assertEquals(deserializedStats.getStats().totalUploadsFailed, stats.getStats().totalUploadsFailed); - assertEquals(deserializedStats.getStats().totalUploadsSucceeded, stats.getStats().totalUploadsSucceeded); - assertEquals(deserializedStats.getStats().rejectionCount, stats.getStats().rejectionCount); - assertEquals(deserializedStats.getStats().consecutiveFailuresCount, stats.getStats().consecutiveFailuresCount); - assertEquals(deserializedStats.getStats().uploadBytesMovingAverage, stats.getStats().uploadBytesMovingAverage, 0); + RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); + assertEquals(deserializedStats.shardId.toString(), stats.getStats().shardId.toString()); + assertEquals(deserializedStats.refreshTimeLagMs, stats.getStats().refreshTimeLagMs); + assertEquals(deserializedStats.localRefreshNumber, stats.getStats().localRefreshNumber); + assertEquals(deserializedStats.remoteRefreshNumber, stats.getStats().remoteRefreshNumber); + assertEquals(deserializedStats.uploadBytesStarted, stats.getStats().uploadBytesStarted); + assertEquals(deserializedStats.uploadBytesSucceeded, stats.getStats().uploadBytesSucceeded); + assertEquals(deserializedStats.uploadBytesFailed, stats.getStats().uploadBytesFailed); + assertEquals(deserializedStats.totalUploadsStarted, stats.getStats().totalUploadsStarted); + assertEquals(deserializedStats.totalUploadsFailed, stats.getStats().totalUploadsFailed); + assertEquals(deserializedStats.totalUploadsSucceeded, stats.getStats().totalUploadsSucceeded); + assertEquals(deserializedStats.rejectionCount, stats.getStats().rejectionCount); + assertEquals(deserializedStats.consecutiveFailuresCount, stats.getStats().consecutiveFailuresCount); + assertEquals(deserializedStats.uploadBytesMovingAverage, stats.getStats().uploadBytesMovingAverage, 0); assertEquals( - deserializedStats.getStats().uploadBytesPerSecMovingAverage, + deserializedStats.uploadBytesPerSecMovingAverage, stats.getStats().uploadBytesPerSecMovingAverage, 0 ); - assertEquals(deserializedStats.getStats().uploadTimeMovingAverage, stats.getStats().uploadTimeMovingAverage, 0); - assertEquals(deserializedStats.getStats().bytesLag, stats.getStats().bytesLag); + assertEquals(deserializedStats.uploadTimeMovingAverage, stats.getStats().uploadTimeMovingAverage, 0); + assertEquals(deserializedStats.bytesLag, stats.getStats().bytesLag); + assertEquals(deserializedStats.totalDownloadsStarted, stats.getStats().totalDownloadsStarted); + assertEquals(deserializedStats.totalDownloadsSucceeded, stats.getStats().totalDownloadsSucceeded); + assertEquals(deserializedStats.totalDownloadsFailed, stats.getStats().totalDownloadsFailed); + assertEquals(deserializedStats.downloadBytesStarted, stats.getStats().downloadBytesStarted); + assertEquals(deserializedStats.downloadBytesFailed, stats.getStats().downloadBytesFailed); + assertEquals(deserializedStats.downloadBytesSucceeded, stats.getStats().downloadBytesSucceeded); + assertEquals(deserializedStats.lastSuccessfulSegmentDownloadBytes, stats.getStats().lastSuccessfulSegmentDownloadBytes); + assertEquals(deserializedStats.lastDownloadTimestampMs, stats.getStats().lastDownloadTimestampMs); + assertEquals(deserializedStats.downloadBytesMovingAverage, stats.getStats().downloadBytesMovingAverage, 0); + assertEquals(deserializedStats.downloadBytesPerSecMovingAverage, stats.getStats().downloadBytesPerSecMovingAverage, 0); + assertEquals(deserializedStats.downloadTimeMovingAverage, stats.getStats().downloadTimeMovingAverage, 0); } } } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java index 2ee254b10facc..e47000ab9997a 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java @@ -64,7 +64,10 @@ public void testGetShardId() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); assertEquals(shardId, pressureTracker.getShardId()); } @@ -74,7 +77,10 @@ public void testUpdateLocalRefreshSeqNo() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long refreshSeqNo = 2; pressureTracker.updateLocalRefreshSeqNo(refreshSeqNo); @@ -86,7 +92,10 @@ public void testUpdateRemoteRefreshSeqNo() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long refreshSeqNo = 4; pressureTracker.updateRemoteRefreshSeqNo(refreshSeqNo); @@ -98,7 +107,10 @@ public void testUpdateLocalRefreshTimeMs() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long refreshTimeMs = System.nanoTime() / 1_000_000L + randomIntBetween(10, 100); pressureTracker.updateLocalRefreshTimeMs(refreshTimeMs); @@ -110,7 +122,10 @@ public void testUpdateRemoteRefreshTimeMs() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long refreshTimeMs = System.nanoTime() / 1_000_000 + randomIntBetween(10, 100); pressureTracker.updateRemoteRefreshTimeMs(refreshTimeMs); @@ -122,7 +137,10 @@ public void testComputeSeqNoLagOnUpdate() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); int localRefreshSeqNo = randomIntBetween(50, 100); int remoteRefreshSeqNo = randomIntBetween(20, 50); @@ -137,7 +155,10 @@ public void testComputeTimeLagOnUpdate() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long currentLocalRefreshTimeMs = pressureTracker.getLocalRefreshTimeMs(); long currentTimeMs = System.nanoTime() / 1_000_000L; @@ -154,7 +175,10 @@ public void testAddUploadBytesStarted() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); pressureTracker.addUploadBytesStarted(bytesToAdd); @@ -169,7 +193,10 @@ public void testAddUploadBytesFailed() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); pressureTracker.addUploadBytesFailed(bytesToAdd); @@ -184,7 +211,10 @@ public void testAddUploadBytesSucceeded() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); pressureTracker.addUploadBytesSucceeded(bytesToAdd); @@ -199,7 +229,10 @@ public void testGetInflightUploadBytes() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); long bytesStarted = randomLongBetween(10000, 100000); long bytesSucceeded = randomLongBetween(1000, 10000); @@ -215,7 +248,10 @@ public void testIncrementTotalUploadsStarted() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsStarted(); assertEquals(1, pressureTracker.getTotalUploadsStarted()); @@ -228,7 +264,10 @@ public void testIncrementTotalUploadsFailed() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsFailed(); assertEquals(1, pressureTracker.getTotalUploadsFailed()); @@ -241,7 +280,10 @@ public void testIncrementTotalUploadSucceeded() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsSucceeded(); assertEquals(1, pressureTracker.getTotalUploadsSucceeded()); @@ -254,7 +296,10 @@ public void testGetInflightUploads() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsStarted(); assertEquals(1, pressureTracker.getInflightUploads()); @@ -271,7 +316,10 @@ public void testIncrementRejectionCount() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); pressureTracker.incrementRejectionCount(); assertEquals(1, pressureTracker.getRejectionCount()); @@ -284,7 +332,10 @@ public void testGetConsecutiveFailureCount() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsFailed(); assertEquals(1, pressureTracker.getConsecutiveFailureCount()); @@ -299,7 +350,10 @@ public void testComputeBytesLag() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); // Create local file size map @@ -328,7 +382,10 @@ public void testIsUploadBytesAverageReady() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); assertFalse(pressureTracker.isUploadBytesAverageReady()); @@ -355,7 +412,10 @@ public void testIsUploadBytesPerSecAverageReady() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); assertFalse(pressureTracker.isUploadBytesPerSecAverageReady()); @@ -382,7 +442,10 @@ public void testIsUploadTimeMsAverageReady() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); assertFalse(pressureTracker.isUploadTimeMsAverageReady()); @@ -468,7 +531,10 @@ private RemoteRefreshSegmentTracker constructTracker() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize(), + pressureSettings.getDownloadBytesMovingAverageWindowSize(), + pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getDownloadTimeMovingAverageWindowSize() ); segmentPressureTracker.incrementTotalUploadsFailed(); segmentPressureTracker.addUploadTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); From 97b733afe981566b9246754304285e8b24548299 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:36:50 +0530 Subject: [PATCH 02/36] Removed unused loggers and updated upload stats field names Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/stats/RemoteStoreStats.java | 32 +++++++---------- .../stats/RemoteStoreStatsTestHelper.java | 36 +++++++++---------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index 86a0c53732f69..a51608fadb73f 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -8,13 +8,11 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.logging.Loggers; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -33,18 +31,14 @@ public class RemoteStoreStats implements Writeable, ToXContentFragment { private final ShardRouting currentRouting; - private final Logger logger; - public RemoteStoreStats(RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats, ShardRouting currentRouting) { this.remoteSegmentShardStats = remoteSegmentUploadShardStats; this.currentRouting = currentRouting; - this.logger = Loggers.getLogger(getClass(), this.currentRouting.getIndexName()); } public RemoteStoreStats(StreamInput in) throws IOException { this.remoteSegmentShardStats = in.readOptionalWriteable(RemoteRefreshSegmentTracker.Stats::new); this.currentRouting = new ShardRouting(in); - this.logger = Loggers.getLogger(getClass(), this.currentRouting.getIndexName()); } public RemoteRefreshSegmentTracker.Stats getStats() { @@ -94,31 +88,31 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (currentRouting.primary()) { builder.startObject(SubFields.UPLOAD) - .field(UploadFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) - .field(UploadFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) - .field(UploadFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) - .field(UploadFields.REFRESH_LAG, remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber) - .field(UploadFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) - .field(UploadFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) - .field(UploadFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); - builder.startObject(UploadFields.TOTAL_REMOTE_REFRESH) + .field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) + .field(UploadStatsFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) + .field(UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) + .field(UploadStatsFields.REFRESH_LAG, remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber) + .field(UploadStatsFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) + .field(UploadStatsFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) + .field(UploadStatsFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); + builder.startObject(UploadStatsFields.TOTAL_REMOTE_REFRESH) .field(SubFields.STARTED, remoteSegmentShardStats.totalUploadsStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalUploadsSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.totalUploadsFailed); builder.endObject(); - builder.startObject(UploadFields.TOTAL_UPLOADS_IN_BYTES) + builder.startObject(UploadStatsFields.TOTAL_UPLOADS_IN_BYTES) .field(SubFields.STARTED, remoteSegmentShardStats.uploadBytesStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.uploadBytesSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.uploadBytesFailed); builder.endObject(); - builder.startObject(UploadFields.REMOTE_REFRESH_SIZE_IN_BYTES) + builder.startObject(UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES) .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulRemoteRefreshBytes) .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesMovingAverage); builder.endObject(); - builder.startObject(UploadFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC) + builder.startObject(UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC) .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesPerSecMovingAverage); builder.endObject(); - builder.startObject(UploadFields.REMOTE_REFRESH_LATENCY_IN_MILLIS) + builder.startObject(UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS) .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadTimeMovingAverage); builder.endObject(); builder.endObject(); @@ -155,7 +149,7 @@ static final class RoutingFields { /** * Fields for remote store stats response */ - static final class UploadFields { + static final class UploadStatsFields { static final String SHARD_ID = "shard_id"; /** diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index 47c2b4cc11683..dfc6978bfd8bd 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -8,8 +8,6 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; @@ -103,59 +101,59 @@ static void compareStatsResponse(Map statsObject, RemoteRefreshS assertTrue(segmentUploads.isEmpty()); } else { assertTrue(segmentDownloads.isEmpty()); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.LOCAL_REFRESH_TIMESTAMP), + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.LOCAL_REFRESH_TIMESTAMP), (int) pressureTrackerStats.localRefreshClockTimeMs ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_TIMESTAMP), + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_TIMESTAMP), (int) pressureTrackerStats.remoteRefreshClockTimeMs ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.REFRESH_TIME_LAG_IN_MILLIS), + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS), (int) pressureTrackerStats.refreshTimeLagMs ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.REFRESH_LAG), + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_LAG), (int) (pressureTrackerStats.localRefreshNumber - pressureTrackerStats.remoteRefreshNumber) ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.BYTES_LAG), (int) pressureTrackerStats.bytesLag); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.BYTES_LAG), (int) pressureTrackerStats.bytesLag); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.BACKPRESSURE_REJECTION_COUNT), + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.BACKPRESSURE_REJECTION_COUNT), (int) pressureTrackerStats.rejectionCount ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadFields.CONSECUTIVE_FAILURE_COUNT), + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.CONSECUTIVE_FAILURE_COUNT), (int) pressureTrackerStats.consecutiveFailuresCount ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), (int) pressureTrackerStats.uploadBytesStarted ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), (int) pressureTrackerStats.uploadBytesSucceeded ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), (int) pressureTrackerStats.uploadBytesFailed ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), pressureTrackerStats.uploadBytesMovingAverage ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), (int) pressureTrackerStats.lastSuccessfulRemoteRefreshBytes ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), pressureTrackerStats.uploadBytesPerSecMovingAverage ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), (int) pressureTrackerStats.totalUploadsStarted ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.SUCCEEDED), + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.SUCCEEDED), (int) pressureTrackerStats.totalUploadsSucceeded ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), + assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), (int) pressureTrackerStats.totalUploadsFailed ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadFields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), pressureTrackerStats.uploadTimeMovingAverage ); } From e667270281af43dc006a4c5f7b0f0e0744b62f22 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:08:06 +0530 Subject: [PATCH 03/36] Fixing spotless errors Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 10 +- .../remotestore/stats/RemoteStoreStats.java | 6 +- .../stats/RemoteStoreStatsResponse.java | 14 +- .../TransportRemoteStoreStatsAction.java | 1 - .../RemoteRefreshSegmentPressureSettings.java | 5 +- .../remote/RemoteRefreshSegmentTracker.java | 4 +- .../opensearch/index/shard/IndexShard.java | 9 +- .../stats/RemoteStoreStatsResponseTests.java | 2 +- .../stats/RemoteStoreStatsTestHelper.java | 167 +++++++++++++----- .../stats/RemoteStoreStatsTests.java | 6 +- 10 files changed, 155 insertions(+), 69 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 775128d9ff9eb..b76488df606cd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -87,7 +87,7 @@ public void testStatsResponseFromAllNodes() { .filter(stat -> indexShardId.equals(stat.getStats().shardId.toString())) .collect(Collectors.toList()); assertEquals(2, matches.size()); - for (RemoteStoreStats stat: matches) { + for (RemoteStoreStats stat : matches) { ShardRouting routing = stat.getShardRouting(); RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); if (routing.primary()) { @@ -129,13 +129,11 @@ public void testStatsResponseAllShards() { ); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - remoteStoreStatsRequestBuilder = client(node).admin() - .cluster() - .prepareRemoteStoreStats(INDEX_NAME, null); + remoteStoreStatsRequestBuilder = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, null); response = remoteStoreStatsRequestBuilder.get(); assertEquals(6, response.getSuccessfulShards()); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 6); - for (RemoteStoreStats stat: response.getRemoteStoreStats()) { + for (RemoteStoreStats stat : response.getRemoteStoreStats()) { ShardRouting routing = stat.getShardRouting(); stats = stat.getStats(); if (routing.primary()) { @@ -185,7 +183,7 @@ public void testStatsResponseFromLocalNode() { RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); assertTrue(response.getSuccessfulShards() > 0); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); - for (RemoteStoreStats stat: response.getRemoteStoreStats()) { + for (RemoteStoreStats stat : response.getRemoteStoreStats()) { ShardRouting routing = stat.getShardRouting(); RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); if (routing.primary()) { diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index a51608fadb73f..03045a177dca1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -91,7 +91,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) .field(UploadStatsFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) .field(UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) - .field(UploadStatsFields.REFRESH_LAG, remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber) + .field( + UploadStatsFields.REFRESH_LAG, + remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber + ) .field(UploadStatsFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) .field(UploadStatsFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) .field(UploadStatsFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); @@ -146,6 +149,7 @@ static final class RoutingFields { static final String PRIMARY = "primary"; static final String NODE_ID = "node"; } + /** * Fields for remote store stats response */ diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java index b81258d9bdd41..4df77d5ba5092 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -39,6 +38,7 @@ public class RemoteStoreStatsResponse extends BroadcastResponse { private Map>> indexWiseStats; private final Logger logger; + public RemoteStoreStatsResponse(StreamInput in) throws IOException { super(in); remoteStoreStats = in.readArray(RemoteStoreStats::new, RemoteStoreStats[]::new); @@ -92,14 +92,14 @@ public Map>> groupByIndexAndShards() public Map> groupByShards(Set perIndexStats) { Map> shardStats = new HashMap<>(); Set shardIds = new HashSet<>(); - for(RemoteStoreStats eachShardStats: perIndexStats) { + for (RemoteStoreStats eachShardStats : perIndexStats) { int shardId = eachShardStats.getShardRouting().getId(); shardIds.add(shardId); } - for (Integer shardId: shardIds) { + for (Integer shardId : shardIds) { List stats = new ArrayList<>(); - for (RemoteStoreStats perShardStats: perIndexStats) { + for (RemoteStoreStats perShardStats : perIndexStats) { if (perShardStats.getShardRouting().getId() == shardId) { stats.add(perShardStats); } @@ -119,12 +119,12 @@ public void writeTo(StreamOutput out) throws IOException { protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { groupByIndexAndShards(); builder.startObject("indices"); - for (String indexName: indexWiseStats.keySet()) { + for (String indexName : indexWiseStats.keySet()) { builder.startObject(indexName); builder.startObject("shards"); - for (int shardId: indexWiseStats.get(indexName).keySet()) { + for (int shardId : indexWiseStats.get(indexName).keySet()) { builder.startArray(Integer.toString(shardId)); - for (RemoteStoreStats shardStat: indexWiseStats.get(indexName).get(shardId)) { + for (RemoteStoreStats shardStat : indexWiseStats.get(indexName).get(shardId)) { shardStat.toXContent(builder, params); } builder.endArray(); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java index ead43bf56f713..4ab386339772e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java @@ -22,7 +22,6 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java index 2506d1a683ea0..497748781087a 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java @@ -191,7 +191,10 @@ public RemoteRefreshSegmentPressureSettings( DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, remoteUploadPressureService::updateDownloadBytesPerSecMovingAverageWindowSize ); - clusterSettings.addSettingsUpdateConsumer(DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, this::setDownloadBytesPerSecMovingAverageWindowSize); + clusterSettings.addSettingsUpdateConsumer( + DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, + this::setDownloadBytesPerSecMovingAverageWindowSize + ); this.downloadTimeMovingAverageWindowSize = DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE.get(settings); clusterSettings.addSettingsUpdateConsumer( diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index 4d849bff30e63..c87cf3961fc02 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -562,7 +562,9 @@ public void addUploadBytesPerSec(long bytesPerSec) { */ void updateDownloadBytesPerSecMovingAverageWindowSize(int updatedSize) { synchronized (downloadBytesPerSecMutex) { - this.downloadBytesPerSecMovingAverageReference.set(this.downloadBytesPerSecMovingAverageReference.get().copyWithSize(updatedSize)); + this.downloadBytesPerSecMovingAverageReference.set( + this.downloadBytesPerSecMovingAverageReference.get().copyWithSize(updatedSize) + ); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index a974158510cc7..74d1ca19a1bf2 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4635,7 +4635,8 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re null, uploadedSegments, remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), - overrideLocal); + overrideLocal + ); if (refreshLevelSegmentSync && remoteSegmentMetadata != null) { try ( @@ -4820,7 +4821,11 @@ private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTrac downloadStatsTracker.addDownloadBytesStarted(incomingFileSize); } - private void afterSegmentDownloadCompleted(RemoteRefreshSegmentTracker downloadStatsTracker, long downloadedFileSize, long startTimeInNs) { + private void afterSegmentDownloadCompleted( + RemoteRefreshSegmentTracker downloadStatsTracker, + long downloadedFileSize, + long startTimeInNs + ) { long currentTime = System.currentTimeMillis(); downloadStatsTracker.updateLastDownloadTimestampMs(currentTime); downloadStatsTracker.incrementTotalDownloadsSucceeded(); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java index 65baefe8c2c7d..d700148d19bca 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java @@ -67,7 +67,7 @@ public void testSerialization() throws Exception { assertEquals(metadataShardsObject.get("failed"), 0); Map indicesObject = (Map) jsonResponseObject.get("indices"); assertTrue(indicesObject.containsKey("index")); - Map shardsObject = (Map)((Map) indicesObject.get("index")).get("shards"); + Map shardsObject = (Map) ((Map) indicesObject.get("index")).get("shards"); ArrayList> perShardNumberObject = (ArrayList>) shardsObject.get("0"); assertEquals(perShardNumberObject.size(), 1); compareStatsResponse(perShardNumberObject.get(0), pressureTrackerStats, primaryShardRouting); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index dfc6978bfd8bd..975692875ac38 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -16,7 +16,6 @@ import java.util.Map; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.opensearch.test.OpenSearchTestCase.assertEquals; import static org.opensearch.test.OpenSearchTestCase.randomAlphaOfLength; @@ -26,134 +25,214 @@ */ public class RemoteStoreStatsTestHelper { static RemoteRefreshSegmentTracker.Stats createPressureTrackerStats(ShardId shardId) { - return new RemoteRefreshSegmentTracker.Stats(shardId, - 101, 102, 100, - 3, 10, 2, - 10, 5, 5, - 10, 10, 5, - 5, 5, 5, - 5, 5, 5, - 3, 2, 5, - 2, 3, 4, - 9, 3, 4, - 9, 5); + return new RemoteRefreshSegmentTracker.Stats( + shardId, + 101, + 102, + 100, + 3, + 10, + 2, + 10, + 5, + 5, + 10, + 10, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 2, + 5, + 2, + 3, + 4, + 9, + 3, + 4, + 9, + 5 + ); } static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); } - static void compareStatsResponse(Map statsObject, RemoteRefreshSegmentTracker.Stats pressureTrackerStats, + static void compareStatsResponse( + Map statsObject, + RemoteRefreshSegmentTracker.Stats pressureTrackerStats, ShardRouting routing ) { - assertEquals(((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.NODE_ID), routing.currentNodeId()); - assertEquals(((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.STATE), routing.state().toString()); - assertEquals(((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.PRIMARY), routing.primary()); + assertEquals( + ((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.NODE_ID), + routing.currentNodeId() + ); + assertEquals( + ((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.STATE), + routing.state().toString() + ); + assertEquals( + ((Map) statsObject.get(RemoteStoreStats.Fields.ROUTING)).get(RemoteStoreStats.RoutingFields.PRIMARY), + routing.primary() + ); Map segment = ((Map) statsObject.get(RemoteStoreStats.Fields.SEGMENT)); Map segmentDownloads = ((Map) segment.get(RemoteStoreStats.SubFields.DOWNLOAD)); Map segmentUploads = ((Map) segment.get(RemoteStoreStats.SubFields.UPLOAD)); - if(!routing.primary()) { - assertEquals(segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP), + if (!routing.primary()) { + assertEquals( + segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP), (int) pressureTrackerStats.lastDownloadTimestampMs ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get(RemoteStoreStats.SubFields.STARTED), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get( + RemoteStoreStats.SubFields.STARTED + ), (int) pressureTrackerStats.totalDownloadsStarted ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get(RemoteStoreStats.SubFields.SUCCEEDED), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get( + RemoteStoreStats.SubFields.SUCCEEDED + ), (int) pressureTrackerStats.totalDownloadsSucceeded ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get(RemoteStoreStats.SubFields.FAILED), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get( + RemoteStoreStats.SubFields.FAILED + ), (int) pressureTrackerStats.totalDownloadsFailed ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get( + RemoteStoreStats.SubFields.STARTED + ), (int) pressureTrackerStats.downloadBytesStarted ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get( + RemoteStoreStats.SubFields.SUCCEEDED + ), (int) pressureTrackerStats.downloadBytesSucceeded ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get( + RemoteStoreStats.SubFields.FAILED + ), (int) pressureTrackerStats.downloadBytesFailed ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( + RemoteStoreStats.SubFields.LAST_SUCCESSFUL + ), (int) pressureTrackerStats.lastSuccessfulSegmentDownloadBytes ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( + RemoteStoreStats.SubFields.MOVING_AVG + ), pressureTrackerStats.downloadBytesMovingAverage ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC)).get( + RemoteStoreStats.SubFields.MOVING_AVG + ), pressureTrackerStats.downloadBytesPerSecMovingAverage ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS)).get( + RemoteStoreStats.SubFields.MOVING_AVG + ), pressureTrackerStats.downloadTimeMovingAverage ); assertTrue(segmentUploads.isEmpty()); } else { assertTrue(segmentDownloads.isEmpty()); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.LOCAL_REFRESH_TIMESTAMP), + assertEquals( + segmentUploads.get(RemoteStoreStats.UploadStatsFields.LOCAL_REFRESH_TIMESTAMP), (int) pressureTrackerStats.localRefreshClockTimeMs ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_TIMESTAMP), + assertEquals( + segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_TIMESTAMP), (int) pressureTrackerStats.remoteRefreshClockTimeMs ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS), + assertEquals( + segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS), (int) pressureTrackerStats.refreshTimeLagMs ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_LAG), + assertEquals( + segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_LAG), (int) (pressureTrackerStats.localRefreshNumber - pressureTrackerStats.remoteRefreshNumber) ); assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.BYTES_LAG), (int) pressureTrackerStats.bytesLag); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.BACKPRESSURE_REJECTION_COUNT), + assertEquals( + segmentUploads.get(RemoteStoreStats.UploadStatsFields.BACKPRESSURE_REJECTION_COUNT), (int) pressureTrackerStats.rejectionCount ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.CONSECUTIVE_FAILURE_COUNT), + assertEquals( + segmentUploads.get(RemoteStoreStats.UploadStatsFields.CONSECUTIVE_FAILURE_COUNT), (int) pressureTrackerStats.consecutiveFailuresCount ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.STARTED), + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get( + RemoteStoreStats.SubFields.STARTED + ), (int) pressureTrackerStats.uploadBytesStarted ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.SUCCEEDED), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get( + RemoteStoreStats.SubFields.SUCCEEDED + ), (int) pressureTrackerStats.uploadBytesSucceeded ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get(RemoteStoreStats.SubFields.FAILED), + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get( + RemoteStoreStats.SubFields.FAILED + ), (int) pressureTrackerStats.uploadBytesFailed ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get( + RemoteStoreStats.SubFields.MOVING_AVG + ), pressureTrackerStats.uploadBytesMovingAverage ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get(RemoteStoreStats.SubFields.LAST_SUCCESSFUL), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get( + RemoteStoreStats.SubFields.LAST_SUCCESSFUL + ), (int) pressureTrackerStats.lastSuccessfulRemoteRefreshBytes ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get( + RemoteStoreStats.SubFields.MOVING_AVG + ), pressureTrackerStats.uploadBytesPerSecMovingAverage ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), (int) pressureTrackerStats.totalUploadsStarted ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.SUCCEEDED), + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get( + RemoteStoreStats.SubFields.SUCCEEDED + ), (int) pressureTrackerStats.totalUploadsSucceeded ); - assertEquals(((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), + assertEquals( + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), (int) pressureTrackerStats.totalUploadsFailed ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get(RemoteStoreStats.SubFields.MOVING_AVG), + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get( + RemoteStoreStats.SubFields.MOVING_AVG + ), pressureTrackerStats.uploadTimeMovingAverage ); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 988b662fae040..9a85bba3b34f7 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -88,11 +88,7 @@ public void testSerialization() throws Exception { assertEquals(deserializedStats.rejectionCount, stats.getStats().rejectionCount); assertEquals(deserializedStats.consecutiveFailuresCount, stats.getStats().consecutiveFailuresCount); assertEquals(deserializedStats.uploadBytesMovingAverage, stats.getStats().uploadBytesMovingAverage, 0); - assertEquals( - deserializedStats.uploadBytesPerSecMovingAverage, - stats.getStats().uploadBytesPerSecMovingAverage, - 0 - ); + assertEquals(deserializedStats.uploadBytesPerSecMovingAverage, stats.getStats().uploadBytesPerSecMovingAverage, 0); assertEquals(deserializedStats.uploadTimeMovingAverage, stats.getStats().uploadTimeMovingAverage, 0); assertEquals(deserializedStats.bytesLag, stats.getStats().bytesLag); assertEquals(deserializedStats.totalDownloadsStarted, stats.getStats().totalDownloadsStarted); From 743c09a36b319832d40aeb88e64a586f7d29afde Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 5 Jul 2023 18:58:53 +0530 Subject: [PATCH 04/36] Adding JavaDocs and fixing divide by zero errors on UTs Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remote/RemoteRefreshSegmentTracker.java | 40 ++++++++++++++++++- .../opensearch/index/shard/IndexShard.java | 12 +++--- .../TransportRemoteStoreStatsActionTests.java | 10 ++--- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index c87cf3961fc02..e6a7bd3d70afb 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -52,6 +52,9 @@ public class RemoteRefreshSegmentTracker { */ private volatile long localRefreshClockTimeMs; + /** + * Time in milliseconds for the last successful segment download + */ private volatile long lastDownloadTimestampMs; /** @@ -101,8 +104,19 @@ public class RemoteRefreshSegmentTracker { */ private volatile long uploadBytesSucceeded; + /** + * Cumulative sum of size in bytes of segment files for which download has started. + */ private volatile long downloadBytesStarted; + + /** + * Cumulative sum of size in bytes of segment files for which download has failed. + */ private volatile long downloadBytesFailed; + + /** + * Cumulative sum of size in bytes of segment files for which download has succeeded. + */ private volatile long downloadBytesSucceeded; /** @@ -120,8 +134,19 @@ public class RemoteRefreshSegmentTracker { */ private volatile long totalUploadsSucceeded; + /** + * Cumulative sum of count of segment file downloads that have started. + */ private volatile long totalDownloadsStarted; + + /** + * Cumulative sum of count of segment file downloads that have succeeded. + */ private volatile long totalDownloadsSucceeded; + + /** + * Cumulative sum of count of segment file downloads that have failed. + */ private volatile long totalDownloadsFailed; /** @@ -165,6 +190,10 @@ public class RemoteRefreshSegmentTracker { */ private final Object uploadBytesMutex = new Object(); + /** + * Provides moving average over the last N total size in bytes of segment files downloaded from the remote store. + * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. + */ private final AtomicReference downloadBytesMovingAverageReference; private final Object downloadBytesMutex = new Object(); @@ -177,18 +206,26 @@ public class RemoteRefreshSegmentTracker { private final Object uploadBytesPerSecMutex = new Object(); + /** + * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the remote store. + * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. + */ private final AtomicReference downloadBytesPerSecMovingAverageReference; private final Object downloadBytesPerSecMutex = new Object(); /** - * Provides moving average over the last N overall upload time (in nanos) as part of remote refresh.N is window size. + * Provides moving average over the last N overall upload time (in millis) as part of remote refresh.N is window size. * Wrapped with {@code AtomicReference} for dynamic changes in window size. */ private final AtomicReference uploadTimeMsMovingAverageReference; private final Object uploadTimeMsMutex = new Object(); + /** + * Provides moving average over the last N overall download time (in millis) of segments downloaded from the remote store. + * Wrapped with {@code AtomicReference} for dynamic changes in window size. + */ private final AtomicReference downloadTimeMovingAverageReference; private final Object downloadTimeMutex = new Object(); @@ -851,5 +888,4 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(bytesLag); } } - } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 74d1ca19a1bf2..23fdfe45a4e44 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -44,7 +44,6 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.Term; -import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.ReferenceManager; @@ -4771,14 +4770,14 @@ private String copySegmentFiles( RemoteSegmentStoreDirectory.UploadedSegmentMetadata segmentMetadata = uploadedSegments.get(file); long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - long startTimeInMs = System.currentTimeMillis(); + long startTimeInNs = System.nanoTime(); long segmentSizeInBytes = segmentMetadata.getLength(); beforeSegmentDownload(downloadStatsTracker, segmentSizeInBytes); try { storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); storeDirectory.sync(Collections.singleton(file)); downloadedSegments.add(file); - afterSegmentDownloadCompleted(downloadStatsTracker, segmentSizeInBytes, startTimeInMs); + afterSegmentDownloadCompleted(downloadStatsTracker, segmentSizeInBytes, startTimeInNs); } catch (IOException e) { afterSegmentDownloadFailed(downloadStatsTracker, segmentSizeInBytes); throw e; @@ -4826,15 +4825,14 @@ private void afterSegmentDownloadCompleted( long downloadedFileSize, long startTimeInNs ) { - long currentTime = System.currentTimeMillis(); - downloadStatsTracker.updateLastDownloadTimestampMs(currentTime); + long currentTimeInNs = System.nanoTime(); + downloadStatsTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); downloadStatsTracker.incrementTotalDownloadsSucceeded(); downloadStatsTracker.addDownloadBytes(downloadedFileSize); downloadStatsTracker.addDownloadBytesSucceeded(downloadedFileSize); - long timeTakenInMS = currentTime - startTimeInNs; + long timeTakenInMS = Math.max(1, TimeValue.nsecToMSec(currentTimeInNs - startTimeInNs)); downloadStatsTracker.addDownloadTime(timeTakenInMS); downloadStatsTracker.addDownloadBytesPerSec((downloadedFileSize * 1_000L) / timeTakenInMS); - } private void afterSegmentDownloadFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index 086c5331fbc80..939a9efa6790e 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -108,7 +108,7 @@ public void tearDown() throws Exception { clusterService.close(); } - public void testOnlyPrimaryShards() throws Exception { + public void testAllShardCopies() throws Exception { FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); RoutingTable routingTable = RoutingTable.builder().addAsNew(remoteStoreIndexMetadata).build(); Metadata metadata = Metadata.builder().put(remoteStoreIndexMetadata, false).build(); @@ -125,7 +125,7 @@ public void testOnlyPrimaryShards() throws Exception { new String[] { INDEX.getName() } ); - assertEquals(shardsIterator.size(), 2); + assertEquals(shardsIterator.size(), 4); } public void testOnlyLocalShards() throws Exception { @@ -153,10 +153,10 @@ public void testOnlyLocalShards() throws Exception { remoteStoreStatsRequest.local(true); ShardsIterator shardsIterator = statsAction.shards(clusterService.state(), remoteStoreStatsRequest, concreteIndices); - assertEquals(shardsIterator.size(), 1); + assertEquals(shardsIterator.size(), 2); } - public void testOnlyRemoteStoreEnabledShards() throws Exception { + public void testOnlyRemoteStoreEnabledShardCopies() throws Exception { FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); Index NEW_INDEX = new Index("newIndex", "newUUID"); IndexMetadata indexMetadataWithoutRemoteStore = IndexMetadata.builder(NEW_INDEX.getName()) @@ -189,6 +189,6 @@ public void testOnlyRemoteStoreEnabledShards() throws Exception { new String[] { INDEX.getName() } ); - assertEquals(shardsIterator.size(), 2); + assertEquals(shardsIterator.size(), 4); } } From ac3fbc014b5c95283ff86fec32a72a65f66c8323 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 6 Jul 2023 18:55:58 +0530 Subject: [PATCH 05/36] Excluding stats publishing on non-remote store indices Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../main/java/org/opensearch/index/shard/IndexShard.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 23fdfe45a4e44..1b590f7f01468 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4816,6 +4816,9 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo } private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTracker, long incomingFileSize) { + if (!isRemoteStoreEnabled()) { + return; + } downloadStatsTracker.incrementTotalDownloadsStarted(); downloadStatsTracker.addDownloadBytesStarted(incomingFileSize); } @@ -4825,6 +4828,9 @@ private void afterSegmentDownloadCompleted( long downloadedFileSize, long startTimeInNs ) { + if (!isRemoteStoreEnabled()) { + return; + } long currentTimeInNs = System.nanoTime(); downloadStatsTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); downloadStatsTracker.incrementTotalDownloadsSucceeded(); @@ -4836,6 +4842,9 @@ private void afterSegmentDownloadCompleted( } private void afterSegmentDownloadFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { + if (!isRemoteStoreEnabled()) { + return; + } downloadStatsTracker.incrementTotalDownloadsFailed(); downloadStatsTracker.addDownloadBytesFailed(failedFileSize); } From b42912f89d5a5bcfafde356446cde7a148ea6f47 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 6 Jul 2023 18:55:58 +0530 Subject: [PATCH 06/36] Excluding stats publishing on non-remote store indices Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../main/java/org/opensearch/index/shard/IndexShard.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 1b590f7f01468..721e2f49e8ee3 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4816,7 +4816,7 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo } private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTracker, long incomingFileSize) { - if (!isRemoteStoreEnabled()) { + if (!indexSettings.isRemoteStoreEnabled()) { return; } downloadStatsTracker.incrementTotalDownloadsStarted(); @@ -4828,7 +4828,7 @@ private void afterSegmentDownloadCompleted( long downloadedFileSize, long startTimeInNs ) { - if (!isRemoteStoreEnabled()) { + if (!indexSettings.isRemoteStoreEnabled()) { return; } long currentTimeInNs = System.nanoTime(); @@ -4842,7 +4842,7 @@ private void afterSegmentDownloadCompleted( } private void afterSegmentDownloadFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { - if (!isRemoteStoreEnabled()) { + if (!indexSettings.isRemoteStoreEnabled()) { return; } downloadStatsTracker.incrementTotalDownloadsFailed(); From 80cf62b6d56f7bd59befab293887555127ef943b Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:06:02 +0530 Subject: [PATCH 07/36] Removing unused logger Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../cluster/remotestore/stats/RemoteStoreStatsResponse.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java index 4df77d5ba5092..9dfd7115929d9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java @@ -37,12 +37,9 @@ public class RemoteStoreStatsResponse extends BroadcastResponse { private Map>> indexWiseStats; - private final Logger logger; - public RemoteStoreStatsResponse(StreamInput in) throws IOException { super(in); remoteStoreStats = in.readArray(RemoteStoreStats::new, RemoteStoreStats[]::new); - this.logger = Loggers.getLogger(getClass(), "sample-index"); } public RemoteStoreStatsResponse( @@ -54,7 +51,6 @@ public RemoteStoreStatsResponse( ) { super(totalShards, successfulShards, failedShards, shardFailures); this.remoteStoreStats = shards; - this.logger = Loggers.getLogger(getClass(), "sample-index"); } public RemoteStoreStats[] getRemoteStoreStats() { @@ -89,7 +85,7 @@ public Map>> groupByIndexAndShards() return indexWiseStats; } - public Map> groupByShards(Set perIndexStats) { + private Map> groupByShards(Set perIndexStats) { Map> shardStats = new HashMap<>(); Set shardIds = new HashSet<>(); for (RemoteStoreStats eachShardStats : perIndexStats) { From d82657939af23948f7b4b86983332386ec86d699 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 11 Jul 2023 15:58:04 +0530 Subject: [PATCH 08/36] Addressing comments Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/stats/RemoteStoreStats.java | 51 ++- .../stats/RemoteStoreStatsResponse.java | 67 +--- .../TransportRemoteStoreStatsAction.java | 2 - .../common/settings/ClusterSettings.java | 3 - .../RemoteRefreshSegmentPressureService.java | 15 - .../RemoteRefreshSegmentPressureSettings.java | 75 ---- .../remote/RemoteRefreshSegmentTracker.java | 90 ++--- .../opensearch/index/shard/IndexShard.java | 50 +-- .../stats/RemoteStoreStatsTestHelper.java | 6 +- .../RemoteRefreshSegmentTrackerTests.java | 340 +++++++++++++----- 10 files changed, 362 insertions(+), 337 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index 03045a177dca1..c16db06e517de 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -29,16 +29,16 @@ public class RemoteStoreStats implements Writeable, ToXContentFragment { private final RemoteRefreshSegmentTracker.Stats remoteSegmentShardStats; - private final ShardRouting currentRouting; + private final ShardRouting shardRouting; - public RemoteStoreStats(RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats, ShardRouting currentRouting) { + public RemoteStoreStats(RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats, ShardRouting shardRouting) { this.remoteSegmentShardStats = remoteSegmentUploadShardStats; - this.currentRouting = currentRouting; + this.shardRouting = shardRouting; } public RemoteStoreStats(StreamInput in) throws IOException { this.remoteSegmentShardStats = in.readOptionalWriteable(RemoteRefreshSegmentTracker.Stats::new); - this.currentRouting = new ShardRouting(in); + this.shardRouting = new ShardRouting(in); } public RemoteRefreshSegmentTracker.Stats getStats() { @@ -46,22 +46,22 @@ public RemoteRefreshSegmentTracker.Stats getStats() { } public ShardRouting getShardRouting() { - return currentRouting; + return shardRouting; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(Fields.ROUTING); - builder.field(RoutingFields.STATE, currentRouting.state()); - builder.field(RoutingFields.PRIMARY, currentRouting.primary()); - builder.field(RoutingFields.NODE_ID, currentRouting.currentNodeId()); + builder.field(RoutingFields.STATE, shardRouting.state()); + builder.field(RoutingFields.PRIMARY, shardRouting.primary()); + builder.field(RoutingFields.NODE_ID, shardRouting.currentNodeId()); builder.endObject(); builder.startObject(Fields.SEGMENT); - if (!currentRouting.primary()) { + if (!shardRouting.primary()) { builder.startObject(SubFields.DOWNLOAD); builder.field(DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); - builder.startObject(DownloadStatsFields.TOTAL_FILE_DOWNLOADS) + builder.startObject(DownloadStatsFields.TOTAL_FILES_DOWNLOADED) .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); @@ -86,7 +86,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(SubFields.DOWNLOAD); builder.endObject(); } - if (currentRouting.primary()) { + if (shardRouting.primary()) { builder.startObject(SubFields.UPLOAD) .field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) .field(UploadStatsFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) @@ -135,7 +135,7 @@ public String toString() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(remoteSegmentShardStats); - currentRouting.writeTo(out); + shardRouting.writeTo(out); } static final class Fields { @@ -154,8 +154,6 @@ static final class RoutingFields { * Fields for remote store stats response */ static final class UploadStatsFields { - static final String SHARD_ID = "shard_id"; - /** * Lag in terms of bytes b/w local and remote store */ @@ -218,11 +216,34 @@ static final class UploadStatsFields { } static final class DownloadStatsFields { + /** + * Last successful segment download timestamp in milliseconds + */ static final String LAST_DOWNLOAD_TIMESTAMP = "last_download_timestamp"; - static final String TOTAL_FILE_DOWNLOADS = "total_file_downloads"; + + /** + * Total number of segment files downloaded from the remote store for a specific shard + */ + static final String TOTAL_FILES_DOWNLOADED = "total_file_downloads"; + + /** + * Total bytes of segment files downloaded from the remote store for a specific shard + */ static final String TOTAL_FILE_DOWNLOADS_IN_BYTES = "total_file_downloads_in_bytes"; + + /** + * Size of each segment file downloaded from the remote store + */ static final String DOWNLOAD_SIZE_IN_BYTES = "download_size_in_bytes"; + + /** + * Speed (in bytes/sec) for segment file downloads + */ static final String DOWNLOAD_SPEED_IN_BYTES_PER_SEC = "download_speed_in_bytes_per_sec"; + + /** + * Time taken (in millis) for each segment file downloaded + */ static final String DOWNLOAD_LATENCY_IN_MILLIS = "download_latency_in_millis"; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java index 9dfd7115929d9..a7147e896c1c8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java @@ -8,23 +8,19 @@ package org.opensearch.action.admin.cluster.remotestore.stats; -import org.apache.logging.log4j.Logger; import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.logging.Loggers; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; /** * Remote Store stats response @@ -35,76 +31,36 @@ public class RemoteStoreStatsResponse extends BroadcastResponse { private final RemoteStoreStats[] remoteStoreStats; - private Map>> indexWiseStats; - public RemoteStoreStatsResponse(StreamInput in) throws IOException { super(in); remoteStoreStats = in.readArray(RemoteStoreStats::new, RemoteStoreStats[]::new); } public RemoteStoreStatsResponse( - RemoteStoreStats[] shards, + RemoteStoreStats[] remoteStoreStats, int totalShards, int successfulShards, int failedShards, List shardFailures ) { super(totalShards, successfulShards, failedShards, shardFailures); - this.remoteStoreStats = shards; + this.remoteStoreStats = remoteStoreStats; } public RemoteStoreStats[] getRemoteStoreStats() { return this.remoteStoreStats; } - public RemoteStoreStats getAt(int position) { - return remoteStoreStats[position]; - } - public Map>> groupByIndexAndShards() { - if (indexWiseStats != null) { - return indexWiseStats; - } - Set indexNames = new HashSet<>(); - for (RemoteStoreStats shardStat : remoteStoreStats) { - String indexName = shardStat.getShardRouting().getIndexName(); - indexNames.add(indexName); - } - Map>> indexWiseStats = new HashMap<>(); - for (String indexName : indexNames) { - Set perIndexStats = new HashSet<>(); - for (RemoteStoreStats shardStat : remoteStoreStats) { - if (shardStat.getShardRouting().getIndexName().equals(indexName)) { - perIndexStats.add(shardStat); - } - } - indexWiseStats.put(indexName, groupByShards(perIndexStats)); + for (RemoteStoreStats shardStat : remoteStoreStats) { + indexWiseStats.computeIfAbsent(shardStat.getShardRouting().getIndexName(), k -> new HashMap<>()) + .computeIfAbsent(shardStat.getShardRouting().getId(), k -> new ArrayList<>()) + .add(shardStat); } - this.indexWiseStats = indexWiseStats; return indexWiseStats; } - private Map> groupByShards(Set perIndexStats) { - Map> shardStats = new HashMap<>(); - Set shardIds = new HashSet<>(); - for (RemoteStoreStats eachShardStats : perIndexStats) { - int shardId = eachShardStats.getShardRouting().getId(); - shardIds.add(shardId); - } - - for (Integer shardId : shardIds) { - List stats = new ArrayList<>(); - for (RemoteStoreStats perShardStats : perIndexStats) { - if (perShardStats.getShardRouting().getId() == shardId) { - stats.add(perShardStats); - } - } - shardStats.put(shardId, stats); - } - return shardStats; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -113,11 +69,11 @@ public void writeTo(StreamOutput out) throws IOException { @Override protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { - groupByIndexAndShards(); - builder.startObject("indices"); + Map>> indexWiseStats = groupByIndexAndShards(); + builder.startObject(Fields.INDICES); for (String indexName : indexWiseStats.keySet()) { builder.startObject(indexName); - builder.startObject("shards"); + builder.startObject(Fields.SHARDS); for (int shardId : indexWiseStats.get(indexName).keySet()) { builder.startArray(Integer.toString(shardId)); for (RemoteStoreStats shardStat : indexWiseStats.get(indexName).get(shardId)) { @@ -135,4 +91,9 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t public String toString() { return Strings.toString(XContentType.JSON, this, true, false); } + + static final class Fields { + static final String SHARDS = "routing"; + static final String INDICES = "indices"; + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java index 4ab386339772e..7de6aa952c75e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java @@ -50,7 +50,6 @@ public class TransportRemoteStoreStatsAction extends TransportBroadcastByNodeAct private final IndicesService indicesService; - private final ClusterService clusterService; private final RemoteRefreshSegmentPressureService remoteRefreshSegmentPressureService; @Inject @@ -71,7 +70,6 @@ public TransportRemoteStoreStatsAction( RemoteStoreStatsRequest::new, ThreadPool.Names.MANAGEMENT ); - this.clusterService = clusterService; this.indicesService = indicesService; this.remoteRefreshSegmentPressureService = remoteRefreshSegmentPressureService; } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 27f18ccb2ea73..d70ea16cf5fdd 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -652,9 +652,6 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteRefreshSegmentPressureSettings.UPLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, RemoteRefreshSegmentPressureSettings.UPLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, RemoteRefreshSegmentPressureSettings.UPLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, - RemoteRefreshSegmentPressureSettings.DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, - RemoteRefreshSegmentPressureSettings.DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, - RemoteRefreshSegmentPressureSettings.DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, // Related to monitoring of task cancellation TaskCancellationMonitoringSettings.IS_ENABLED_SETTING, diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java index dc0c153f72e2d..280381a7b6109 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java @@ -78,9 +78,6 @@ public void afterIndexShardCreated(IndexShard indexShard) { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ) ); @@ -137,18 +134,6 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateUploadTimeMsMovingAverageWindowSize, updatedSize); } - void updateDownloadBytesMovingAverageWindowSize(int updatedSize) { - updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateDownloadBytesPerSecMovingAverageWindowSize, updatedSize); - } - - void updateDownloadBytesPerSecMovingAverageWindowSize(int updatedSize) { - updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateDownloadBytesMovingAverageWindowSize, updatedSize); - } - - void updateDownloadTimeMovingAverageWindowSize(int updatedSize) { - updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateDownloadTimeMsMovingAverageWindowSize, updatedSize); - } - void updateMovingAverageWindowSize(BiConsumer biConsumer, int updatedSize) { trackerMap.values().forEach(tracker -> biConsumer.accept(tracker, updatedSize)); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java index 497748781087a..2ec03af7f7a63 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java @@ -29,9 +29,6 @@ private static class Defaults { private static final int UPLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE = 20; private static final int UPLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE = 20; private static final int UPLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE = 20; - private static final int DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE = 20; - private static final int DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE = 20; - private static final int DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE = 20; private static final int MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE = 5; } @@ -90,30 +87,6 @@ private static class Defaults { Setting.Property.NodeScope ); - public static final Setting DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE = Setting.intSetting( - "remote_store.segment.pressure.download_size_in_bytes_moving_average_window_size", - Defaults.DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, - Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE, - Setting.Property.Dynamic, - Setting.Property.NodeScope - ); - - public static final Setting DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE = Setting.intSetting( - "remote_store.segment.pressure.download_speed_in_bytes_per_sec_moving_average_window_size", - Defaults.DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, - Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE, - Setting.Property.Dynamic, - Setting.Property.NodeScope - ); - - public static final Setting DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE = Setting.intSetting( - "remote_store.segment.pressure.download_time_moving_average_window_size", - Defaults.DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, - Defaults.MOVING_AVERAGE_WINDOW_SIZE_MIN_VALUE, - Setting.Property.Dynamic, - Setting.Property.NodeScope - ); - private volatile boolean remoteRefreshSegmentPressureEnabled; private volatile long minRefreshSeqNoLagLimit; @@ -178,30 +151,6 @@ public RemoteRefreshSegmentPressureSettings( remoteUploadPressureService::updateUploadTimeMsMovingAverageWindowSize ); clusterSettings.addSettingsUpdateConsumer(UPLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, this::setUploadTimeMovingAverageWindowSize); - - this.downloadBytesMovingAverageWindowSize = DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE.get(settings); - clusterSettings.addSettingsUpdateConsumer( - DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, - remoteUploadPressureService::updateDownloadBytesMovingAverageWindowSize - ); - clusterSettings.addSettingsUpdateConsumer(DOWNLOAD_BYTES_MOVING_AVERAGE_WINDOW_SIZE, this::setDownloadBytesMovingAverageWindowSize); - - this.downloadBytesPerSecMovingAverageWindowSize = DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE.get(settings); - clusterSettings.addSettingsUpdateConsumer( - DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, - remoteUploadPressureService::updateDownloadBytesPerSecMovingAverageWindowSize - ); - clusterSettings.addSettingsUpdateConsumer( - DOWNLOAD_BYTES_PER_SEC_MOVING_AVERAGE_WINDOW_SIZE, - this::setDownloadBytesPerSecMovingAverageWindowSize - ); - - this.downloadTimeMovingAverageWindowSize = DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE.get(settings); - clusterSettings.addSettingsUpdateConsumer( - DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, - remoteUploadPressureService::updateDownloadTimeMovingAverageWindowSize - ); - clusterSettings.addSettingsUpdateConsumer(DOWNLOAD_TIME_MOVING_AVERAGE_WINDOW_SIZE, this::setDownloadTimeMovingAverageWindowSize); } public boolean isRemoteRefreshSegmentPressureEnabled() { @@ -267,28 +216,4 @@ public int getUploadTimeMovingAverageWindowSize() { public void setUploadTimeMovingAverageWindowSize(int uploadTimeMovingAverageWindowSize) { this.uploadTimeMovingAverageWindowSize = uploadTimeMovingAverageWindowSize; } - - public int getDownloadBytesMovingAverageWindowSize() { - return downloadBytesMovingAverageWindowSize; - } - - public void setDownloadBytesMovingAverageWindowSize(int downloadBytesMovingAverageWindowSize) { - this.downloadBytesMovingAverageWindowSize = downloadBytesMovingAverageWindowSize; - } - - public int getDownloadBytesPerSecMovingAverageWindowSize() { - return downloadBytesPerSecMovingAverageWindowSize; - } - - public void setDownloadBytesPerSecMovingAverageWindowSize(int downloadBytesPerSecMovingAverageWindowSize) { - this.downloadBytesPerSecMovingAverageWindowSize = downloadBytesPerSecMovingAverageWindowSize; - } - - public int getDownloadTimeMovingAverageWindowSize() { - return downloadBytesPerSecMovingAverageWindowSize; - } - - public void setDownloadTimeMovingAverageWindowSize(int downloadBytesPerSecMovingAverageWindowSize) { - this.downloadBytesPerSecMovingAverageWindowSize = downloadBytesPerSecMovingAverageWindowSize; - } } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index e6a7bd3d70afb..9b69a228d055e 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -228,16 +228,15 @@ public class RemoteRefreshSegmentTracker { */ private final AtomicReference downloadTimeMovingAverageReference; + private final int SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE = 20; + private final Object downloadTimeMutex = new Object(); public RemoteRefreshSegmentTracker( ShardId shardId, int uploadBytesMovingAverageWindowSize, int uploadBytesPerSecMovingAverageWindowSize, - int uploadTimeMsMovingAverageWindowSize, - int downloadBytesMovingAverageWindowSize, - int downloadBytesPerSecMovingAverageWindowSize, - int downloadTimeMovingAverageWindowSize + int uploadTimeMsMovingAverageWindowSize ) { this.shardId = shardId; // Both the local refresh time and remote refresh time are set with current time to give consistent view of time lag when it arises. @@ -250,9 +249,9 @@ public RemoteRefreshSegmentTracker( uploadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesMovingAverageWindowSize)); uploadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesPerSecMovingAverageWindowSize)); uploadTimeMsMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadTimeMsMovingAverageWindowSize)); - downloadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(downloadBytesMovingAverageWindowSize)); - downloadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(downloadBytesPerSecMovingAverageWindowSize)); - downloadTimeMovingAverageReference = new AtomicReference<>(new MovingAverage(downloadTimeMovingAverageWindowSize)); + downloadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE)); + downloadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE)); + downloadTimeMovingAverageReference = new AtomicReference<>(new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE)); latestLocalFileNameLengthMap = new HashMap<>(); } @@ -450,8 +449,8 @@ public long getTotalDownloadsStarted() { return totalDownloadsStarted; } - public void incrementTotalDownloadsStarted() { - totalDownloadsStarted += 1; + public void addTotalDownloadsStarted(long totalFiles) { + totalDownloadsStarted += totalFiles; } public long getTotalDownloadsFailed() { @@ -557,7 +556,7 @@ boolean isDownloadBytesAverageReady() { } double getDownloadBytesAverage() { - return uploadBytesMovingAverageReference.get().getAverage(); + return downloadBytesMovingAverageReference.get().getAverage(); } public void addDownloadBytes(long size) { @@ -567,17 +566,6 @@ public void addDownloadBytes(long size) { } } - /** - * Updates the window size for data collection of upload bytes. This also resets any data collected so far. - * - * @param updatedSize the updated size - */ - void updateDownloadBytesMovingAverageWindowSize(int updatedSize) { - synchronized (downloadBytesMutex) { - this.downloadBytesMovingAverageReference.set(this.downloadBytesMovingAverageReference.get().copyWithSize(updatedSize)); - } - } - boolean isUploadBytesPerSecAverageReady() { return uploadBytesPerSecMovingAverageReference.get().isReady(); } @@ -592,25 +580,12 @@ public void addUploadBytesPerSec(long bytesPerSec) { } } - /** - * Updates the window size for data collection of upload bytes per second. This also resets any data collected so far. - * - * @param updatedSize the updated size - */ - void updateDownloadBytesPerSecMovingAverageWindowSize(int updatedSize) { - synchronized (downloadBytesPerSecMutex) { - this.downloadBytesPerSecMovingAverageReference.set( - this.downloadBytesPerSecMovingAverageReference.get().copyWithSize(updatedSize) - ); - } - } - boolean isDownloadBytesPerSecAverageReady() { - return uploadBytesPerSecMovingAverageReference.get().isReady(); + return downloadBytesPerSecMovingAverageReference.get().isReady(); } double getDownloadBytesPerSecAverage() { - return uploadBytesPerSecMovingAverageReference.get().getAverage(); + return downloadBytesPerSecMovingAverageReference.get().getAverage(); } public void addDownloadBytesPerSec(long bytesPerSec) { @@ -669,17 +644,6 @@ public void addDownloadTime(long timeMs) { } } - /** - * Updates the window size for data collection of upload time (ms). This also resets any data collected so far. - * - * @param updatedSize the updated size - */ - void updateDownloadTimeMsMovingAverageWindowSize(int updatedSize) { - synchronized (downloadTimeMutex) { - this.downloadTimeMovingAverageReference.set(this.downloadTimeMovingAverageReference.get().copyWithSize(updatedSize)); - } - } - public RemoteRefreshSegmentTracker.Stats stats() { return new RemoteRefreshSegmentTracker.Stats( shardId, @@ -823,32 +787,32 @@ public Stats(StreamInput in) throws IOException { this.localRefreshClockTimeMs = in.readLong(); this.remoteRefreshClockTimeMs = in.readLong(); this.refreshTimeLagMs = in.readLong(); - this.lastDownloadTimestampMs = in.readLong(); this.localRefreshNumber = in.readLong(); this.remoteRefreshNumber = in.readLong(); this.uploadBytesStarted = in.readLong(); this.uploadBytesFailed = in.readLong(); this.uploadBytesSucceeded = in.readLong(); - this.downloadBytesStarted = in.readLong(); - this.downloadBytesFailed = in.readLong(); - this.downloadBytesSucceeded = in.readLong(); this.totalUploadsStarted = in.readLong(); this.totalUploadsFailed = in.readLong(); this.totalUploadsSucceeded = in.readLong(); - this.totalDownloadsStarted = in.readLong(); - this.totalDownloadsFailed = in.readLong(); - this.totalDownloadsSucceeded = in.readLong(); this.rejectionCount = in.readLong(); this.consecutiveFailuresCount = in.readLong(); this.lastSuccessfulRemoteRefreshBytes = in.readLong(); this.uploadBytesMovingAverage = in.readDouble(); this.uploadBytesPerSecMovingAverage = in.readDouble(); this.uploadTimeMovingAverage = in.readDouble(); + this.bytesLag = in.readLong(); + this.lastDownloadTimestampMs = in.readLong(); + this.downloadBytesStarted = in.readLong(); + this.downloadBytesFailed = in.readLong(); + this.downloadBytesSucceeded = in.readLong(); + this.totalDownloadsStarted = in.readLong(); + this.totalDownloadsFailed = in.readLong(); + this.totalDownloadsSucceeded = in.readLong(); this.lastSuccessfulSegmentDownloadBytes = in.readLong(); this.downloadBytesMovingAverage = in.readDouble(); this.downloadBytesPerSecMovingAverage = in.readDouble(); this.downloadTimeMovingAverage = in.readDouble(); - this.bytesLag = in.readLong(); } catch (IOException e) { throw e; } @@ -860,32 +824,32 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(localRefreshClockTimeMs); out.writeLong(remoteRefreshClockTimeMs); out.writeLong(refreshTimeLagMs); - out.writeLong(lastDownloadTimestampMs); out.writeLong(localRefreshNumber); out.writeLong(remoteRefreshNumber); out.writeLong(uploadBytesStarted); out.writeLong(uploadBytesFailed); out.writeLong(uploadBytesSucceeded); - out.writeLong(downloadBytesStarted); - out.writeLong(downloadBytesFailed); - out.writeLong(downloadBytesSucceeded); out.writeLong(totalUploadsStarted); out.writeLong(totalUploadsFailed); out.writeLong(totalUploadsSucceeded); - out.writeLong(totalDownloadsStarted); - out.writeLong(totalDownloadsFailed); - out.writeLong(totalDownloadsSucceeded); out.writeLong(rejectionCount); out.writeLong(consecutiveFailuresCount); out.writeLong(lastSuccessfulRemoteRefreshBytes); out.writeDouble(uploadBytesMovingAverage); out.writeDouble(uploadBytesPerSecMovingAverage); out.writeDouble(uploadTimeMovingAverage); + out.writeLong(bytesLag); + out.writeLong(lastDownloadTimestampMs); + out.writeLong(downloadBytesStarted); + out.writeLong(downloadBytesFailed); + out.writeLong(downloadBytesSucceeded); + out.writeLong(totalDownloadsStarted); + out.writeLong(totalDownloadsFailed); + out.writeLong(totalDownloadsSucceeded); out.writeLong(lastSuccessfulSegmentDownloadBytes); out.writeDouble(downloadBytesMovingAverage); out.writeDouble(downloadBytesPerSecMovingAverage); out.writeDouble(downloadTimeMovingAverage); - out.writeLong(bytesLag); } } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 721e2f49e8ee3..9d95dc2e8efa4 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4748,6 +4748,7 @@ public void syncSegmentsFromGivenRemoteSegmentStore( } } + // TODO: Move this method to a generic util class along-with the segment upload code private String copySegmentFiles( Directory storeDirectory, RemoteSegmentStoreDirectory sourceRemoteDirectory, @@ -4758,6 +4759,8 @@ private String copySegmentFiles( ) throws IOException { List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); + List segmentsToDownload = new ArrayList<>(); + long totalSizeOfSegmentsToDownload = 0; String segmentNFile = null; try { Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); @@ -4767,21 +4770,10 @@ private String copySegmentFiles( } } for (String file : uploadedSegments.keySet()) { - RemoteSegmentStoreDirectory.UploadedSegmentMetadata segmentMetadata = uploadedSegments.get(file); long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - long startTimeInNs = System.nanoTime(); - long segmentSizeInBytes = segmentMetadata.getLength(); - beforeSegmentDownload(downloadStatsTracker, segmentSizeInBytes); - try { - storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - storeDirectory.sync(Collections.singleton(file)); - downloadedSegments.add(file); - afterSegmentDownloadCompleted(downloadStatsTracker, segmentSizeInBytes, startTimeInNs); - } catch (IOException e) { - afterSegmentDownloadFailed(downloadStatsTracker, segmentSizeInBytes); - throw e; - } + segmentsToDownload.add(file); + totalSizeOfSegmentsToDownload += uploadedSegments.get(file).getLength(); } else { skippedSegments.add(file); } @@ -4793,6 +4785,18 @@ private String copySegmentFiles( segmentNFile = file; } } + beforeSegmentDownload(downloadStatsTracker, segmentsToDownload.size(), totalSizeOfSegmentsToDownload); + for (String file : segmentsToDownload) { + long startTimeInMs = System.currentTimeMillis(); + try { + storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); + storeDirectory.sync(Collections.singleton(file)); + downloadedSegments.add(file); + afterSegmentDownloadCompleted(downloadStatsTracker, uploadedSegments.get(file).getLength(), startTimeInMs); + } catch (IOException e) { + afterSegmentDownloadFailed(downloadStatsTracker, uploadedSegments.get(file).getLength()); + } + } } finally { logger.info("Downloaded segments here: {}", downloadedSegments); logger.info("Skipped download for segments here: {}", skippedSegments); @@ -4815,28 +4819,34 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo return false; } - private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTracker, long incomingFileSize) { + // TODO: Move the following three methods also to the generic class with `copySegmentFiles` + private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTracker, long totalFiles, long incomingFilesSize) { + // The `copySegmentFiles` method is being used for both + // - `syncSegmentsFromRemoteSegmentStore` (Remote store based shard recovery) + // - `syncSegmentsFromGivenRemoteSegmentStore` (Snapshot inter-op with remote store) + // Since there is a provision to disable remote store in the index that is being restored from a snapshot, + // adding this check to ensure that the stats code path does not get executed if (!indexSettings.isRemoteStoreEnabled()) { return; } - downloadStatsTracker.incrementTotalDownloadsStarted(); - downloadStatsTracker.addDownloadBytesStarted(incomingFileSize); + downloadStatsTracker.addTotalDownloadsStarted(totalFiles); + downloadStatsTracker.addDownloadBytesStarted(incomingFilesSize); } private void afterSegmentDownloadCompleted( RemoteRefreshSegmentTracker downloadStatsTracker, long downloadedFileSize, - long startTimeInNs + long startTimeInMs ) { if (!indexSettings.isRemoteStoreEnabled()) { return; } - long currentTimeInNs = System.nanoTime(); - downloadStatsTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); + long currentTimeInMs = System.currentTimeMillis(); + downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); downloadStatsTracker.incrementTotalDownloadsSucceeded(); downloadStatsTracker.addDownloadBytes(downloadedFileSize); downloadStatsTracker.addDownloadBytesSucceeded(downloadedFileSize); - long timeTakenInMS = Math.max(1, TimeValue.nsecToMSec(currentTimeInNs - startTimeInNs)); + long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); downloadStatsTracker.addDownloadTime(timeTakenInMS); downloadStatsTracker.addDownloadBytesPerSec((downloadedFileSize * 1_000L) / timeTakenInMS); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index 975692875ac38..75696dd3a5cd2 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -91,19 +91,19 @@ static void compareStatsResponse( (int) pressureTrackerStats.lastDownloadTimestampMs ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILES_DOWNLOADED)).get( RemoteStoreStats.SubFields.STARTED ), (int) pressureTrackerStats.totalDownloadsStarted ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILES_DOWNLOADED)).get( RemoteStoreStats.SubFields.SUCCEEDED ), (int) pressureTrackerStats.totalDownloadsSucceeded ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILES_DOWNLOADED)).get( RemoteStoreStats.SubFields.FAILED ), (int) pressureTrackerStats.totalDownloadsFailed diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java index e47000ab9997a..bd33e69679e26 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java @@ -64,10 +64,7 @@ public void testGetShardId() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); assertEquals(shardId, pressureTracker.getShardId()); } @@ -77,10 +74,7 @@ public void testUpdateLocalRefreshSeqNo() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long refreshSeqNo = 2; pressureTracker.updateLocalRefreshSeqNo(refreshSeqNo); @@ -92,10 +86,7 @@ public void testUpdateRemoteRefreshSeqNo() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long refreshSeqNo = 4; pressureTracker.updateRemoteRefreshSeqNo(refreshSeqNo); @@ -107,10 +98,7 @@ public void testUpdateLocalRefreshTimeMs() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long refreshTimeMs = System.nanoTime() / 1_000_000L + randomIntBetween(10, 100); pressureTracker.updateLocalRefreshTimeMs(refreshTimeMs); @@ -122,25 +110,31 @@ public void testUpdateRemoteRefreshTimeMs() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long refreshTimeMs = System.nanoTime() / 1_000_000 + randomIntBetween(10, 100); pressureTracker.updateRemoteRefreshTimeMs(refreshTimeMs); assertEquals(refreshTimeMs, pressureTracker.getRemoteRefreshTimeMs()); } + public void testLastDownloadTimestampMs() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + long currentTimeInMs = System.currentTimeMillis(); + pressureTracker.updateLastDownloadTimestampMs(currentTimeInMs); + assertEquals(currentTimeInMs, pressureTracker.getLastDownloadTimestampMs()); + } + public void testComputeSeqNoLagOnUpdate() { pressureTracker = new RemoteRefreshSegmentTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); int localRefreshSeqNo = randomIntBetween(50, 100); int remoteRefreshSeqNo = randomIntBetween(20, 50); @@ -155,10 +149,7 @@ public void testComputeTimeLagOnUpdate() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long currentLocalRefreshTimeMs = pressureTracker.getLocalRefreshTimeMs(); long currentTimeMs = System.nanoTime() / 1_000_000L; @@ -175,10 +166,7 @@ public void testAddUploadBytesStarted() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); pressureTracker.addUploadBytesStarted(bytesToAdd); @@ -193,10 +181,7 @@ public void testAddUploadBytesFailed() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); pressureTracker.addUploadBytesFailed(bytesToAdd); @@ -211,10 +196,7 @@ public void testAddUploadBytesSucceeded() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); pressureTracker.addUploadBytesSucceeded(bytesToAdd); @@ -224,15 +206,57 @@ public void testAddUploadBytesSucceeded() { assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getUploadBytesSucceeded()); } + public void testAddDownloadBytesStarted() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + long bytesToAdd = randomLongBetween(1000, 1000000); + pressureTracker.addDownloadBytesStarted(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDownloadBytesStarted()); + long moreBytesToAdd = randomLongBetween(1000, 10000); + pressureTracker.addDownloadBytesStarted(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDownloadBytesStarted()); + } + + public void testAddDownloadBytesFailed() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + long bytesToAdd = randomLongBetween(1000, 1000000); + pressureTracker.addDownloadBytesFailed(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDownloadBytesFailed()); + long moreBytesToAdd = randomLongBetween(1000, 10000); + pressureTracker.addDownloadBytesFailed(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDownloadBytesFailed()); + } + + public void testAddDownloadBytesSucceeded() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + long bytesToAdd = randomLongBetween(1000, 1000000); + pressureTracker.addDownloadBytesSucceeded(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDownloadBytesSucceeded()); + long moreBytesToAdd = randomLongBetween(1000, 10000); + pressureTracker.addDownloadBytesSucceeded(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDownloadBytesSucceeded()); + } + public void testGetInflightUploadBytes() { pressureTracker = new RemoteRefreshSegmentTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesStarted = randomLongBetween(10000, 100000); long bytesSucceeded = randomLongBetween(1000, 10000); @@ -243,15 +267,69 @@ public void testGetInflightUploadBytes() { assertEquals(bytesStarted - bytesSucceeded - bytesFailed, pressureTracker.getInflightUploadBytes()); } + public void testGetInFlightDownloadBytes() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + long bytesStarted = randomLongBetween(10000, 100000); + long bytesSucceeded = randomLongBetween(1000, 10000); + long bytesFailed = randomLongBetween(100, 1000); + pressureTracker.addDownloadBytesStarted(bytesStarted); + pressureTracker.addDownloadBytesSucceeded(bytesSucceeded); + pressureTracker.addDownloadBytesFailed(bytesFailed); + assertEquals(bytesStarted - bytesSucceeded - bytesFailed, pressureTracker.getInflightDownloadBytes()); + } + + public void testIncrementTotalDownloadsStarted() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + long firstDownloadBatch = 20; + pressureTracker.addTotalDownloadsStarted(firstDownloadBatch); + assertEquals(firstDownloadBatch, pressureTracker.getTotalDownloadsStarted()); + long secondDownloadBatch = 20; + pressureTracker.addTotalDownloadsStarted(secondDownloadBatch); + assertEquals(firstDownloadBatch + secondDownloadBatch, pressureTracker.getTotalDownloadsStarted()); + } + + public void testIncrementTotalDownloadsFailed() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + pressureTracker.incrementTotalDownloadsFailed(); + assertEquals(1, pressureTracker.getTotalDownloadsFailed()); + pressureTracker.incrementTotalDownloadsFailed(); + assertEquals(2, pressureTracker.getTotalDownloadsFailed()); + } + + public void testIncrementTotalDownloadsSucceeded() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + pressureTracker.incrementTotalDownloadsSucceeded(); + assertEquals(1, pressureTracker.getTotalDownloadsSucceeded()); + pressureTracker.incrementTotalDownloadsSucceeded(); + assertEquals(2, pressureTracker.getTotalDownloadsSucceeded()); + } + public void testIncrementTotalUploadsStarted() { pressureTracker = new RemoteRefreshSegmentTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsStarted(); assertEquals(1, pressureTracker.getTotalUploadsStarted()); @@ -264,10 +342,7 @@ public void testIncrementTotalUploadsFailed() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsFailed(); assertEquals(1, pressureTracker.getTotalUploadsFailed()); @@ -280,10 +355,7 @@ public void testIncrementTotalUploadSucceeded() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsSucceeded(); assertEquals(1, pressureTracker.getTotalUploadsSucceeded()); @@ -296,10 +368,7 @@ public void testGetInflightUploads() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsStarted(); assertEquals(1, pressureTracker.getInflightUploads()); @@ -311,15 +380,29 @@ public void testGetInflightUploads() { assertEquals(0, pressureTracker.getInflightUploads()); } + public void testGetInflightDownloads() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + pressureTracker.addTotalDownloadsStarted(1); + assertEquals(1, pressureTracker.getInflightDownloads()); + pressureTracker.addTotalDownloadsStarted(1); + assertEquals(2, pressureTracker.getInflightDownloads()); + pressureTracker.incrementTotalDownloadsSucceeded(); + assertEquals(1, pressureTracker.getInflightDownloads()); + pressureTracker.incrementTotalDownloadsFailed(); + assertEquals(0, pressureTracker.getInflightDownloads()); + } + public void testIncrementRejectionCount() { pressureTracker = new RemoteRefreshSegmentTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); pressureTracker.incrementRejectionCount(); assertEquals(1, pressureTracker.getRejectionCount()); @@ -332,10 +415,7 @@ public void testGetConsecutiveFailureCount() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); pressureTracker.incrementTotalUploadsFailed(); assertEquals(1, pressureTracker.getConsecutiveFailureCount()); @@ -350,10 +430,7 @@ public void testComputeBytesLag() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); // Create local file size map @@ -382,10 +459,7 @@ public void testIsUploadBytesAverageReady() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); assertFalse(pressureTracker.isUploadBytesAverageReady()); @@ -412,10 +486,7 @@ public void testIsUploadBytesPerSecAverageReady() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); assertFalse(pressureTracker.isUploadBytesPerSecAverageReady()); @@ -442,10 +513,7 @@ public void testIsUploadTimeMsAverageReady() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); assertFalse(pressureTracker.isUploadTimeMsAverageReady()); @@ -467,6 +535,87 @@ public void testIsUploadTimeMsAverageReady() { assertEquals((double) sum / 20, pressureTracker.getUploadTimeMsAverage(), 0.0d); } + public void testIsDownloadBytesAverageReady() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + assertFalse(pressureTracker.isDownloadBytesAverageReady()); + + long sum = 0; + for (int i = 1; i < 20; i++) { + pressureTracker.addDownloadBytes(i); + sum += i; + assertFalse(pressureTracker.isDownloadBytesAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDownloadBytesAverage(), 0.0d); + } + + pressureTracker.addDownloadBytes(20); + sum += 20; + assertTrue(pressureTracker.isDownloadBytesAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDownloadBytesAverage(), 0.0d); + + pressureTracker.addDownloadBytes(100); + sum = sum + 100 - 1; + assertEquals((double) sum / 20, pressureTracker.getDownloadBytesAverage(), 0.0d); + } + + public void testIsDownloadBytesPerSecAverageReady() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + assertFalse(pressureTracker.isDownloadBytesPerSecAverageReady()); + + long sum = 0; + for (int i = 1; i < 20; i++) { + pressureTracker.addDownloadBytesPerSec(i); + sum += i; + assertFalse(pressureTracker.isDownloadBytesPerSecAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); + } + + pressureTracker.addDownloadBytesPerSec(20); + sum += 20; + assertTrue(pressureTracker.isDownloadBytesPerSecAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); + + pressureTracker.addDownloadBytesPerSec(100); + sum = sum + 100 - 1; + assertEquals((double) sum / 20, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); + } + + public void testIsDownloadTimeMsAverageReady() { + pressureTracker = new RemoteRefreshSegmentTracker( + shardId, + pressureSettings.getUploadBytesMovingAverageWindowSize(), + pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), + pressureSettings.getUploadTimeMovingAverageWindowSize() + ); + assertFalse(pressureTracker.isDownloadTimeAverageReady()); + + long sum = 0; + for (int i = 1; i < 20; i++) { + pressureTracker.addDownloadTime(i); + sum += i; + assertFalse(pressureTracker.isDownloadTimeAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDownloadTimeAverage(), 0.0d); + } + + pressureTracker.addDownloadTime(20); + sum += 20; + assertTrue(pressureTracker.isDownloadTimeAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDownloadTimeAverage(), 0.0d); + + pressureTracker.addDownloadTime(100); + sum = sum + 100 - 1; + assertEquals((double) sum / 20, pressureTracker.getDownloadTimeAverage(), 0.0d); + } + /** * Tests whether RemoteRefreshSegmentTracker.Stats object generated correctly from RemoteRefreshSegmentTracker. * */ @@ -489,6 +638,12 @@ public void testStatsObjectCreation() { assertEquals(pressureTracker.getTotalUploadsStarted(), (int) pressureTrackerStats.totalUploadsStarted); assertEquals(pressureTracker.getTotalUploadsSucceeded(), (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals(pressureTracker.getTotalUploadsFailed(), (int) pressureTrackerStats.totalUploadsFailed); + assertEquals(pressureTracker.getLastDownloadTimestampMs(), pressureTrackerStats.lastDownloadTimestampMs); + assertEquals(pressureTracker.getTotalDownloadsStarted(), pressureTrackerStats.totalDownloadsStarted); + assertEquals(pressureTracker.getTotalDownloadsSucceeded(), pressureTrackerStats.totalDownloadsSucceeded); + assertEquals(pressureTracker.getDownloadBytesStarted(), pressureTrackerStats.downloadBytesStarted); + assertEquals(pressureTracker.getDownloadBytesSucceeded(), pressureTrackerStats.downloadBytesStarted); + assertEquals(pressureTracker.getDownloadTimeAverage(), pressureTrackerStats.downloadTimeMovingAverage, 0); } /** @@ -522,6 +677,12 @@ public void testStatsObjectCreationViaStream() throws IOException { assertEquals((int) deserializedStats.totalUploadsStarted, (int) pressureTrackerStats.totalUploadsStarted); assertEquals((int) deserializedStats.totalUploadsSucceeded, (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals((int) deserializedStats.totalUploadsFailed, (int) pressureTrackerStats.totalUploadsFailed); + assertEquals(deserializedStats.lastDownloadTimestampMs, pressureTrackerStats.lastDownloadTimestampMs); + assertEquals(deserializedStats.totalDownloadsStarted, pressureTrackerStats.totalDownloadsStarted); + assertEquals(deserializedStats.totalDownloadsSucceeded, pressureTrackerStats.totalDownloadsSucceeded); + assertEquals(deserializedStats.downloadBytesSucceeded, pressureTrackerStats.downloadBytesSucceeded); + assertEquals(deserializedStats.downloadBytesStarted, pressureTrackerStats.downloadBytesStarted); + assertEquals(deserializedStats.downloadTimeMovingAverage, pressureTrackerStats.downloadTimeMovingAverage, 0); } } } @@ -531,16 +692,19 @@ private RemoteRefreshSegmentTracker constructTracker() { shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize(), - pressureSettings.getDownloadBytesMovingAverageWindowSize(), - pressureSettings.getDownloadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getDownloadTimeMovingAverageWindowSize() + pressureSettings.getUploadTimeMovingAverageWindowSize() ); segmentPressureTracker.incrementTotalUploadsFailed(); segmentPressureTracker.addUploadTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); segmentPressureTracker.addUploadBytes(99); segmentPressureTracker.updateRemoteRefreshTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); segmentPressureTracker.incrementRejectionCount(); + segmentPressureTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); + segmentPressureTracker.addTotalDownloadsStarted(10); + segmentPressureTracker.incrementTotalDownloadsSucceeded(); + segmentPressureTracker.addDownloadBytesStarted(50); + segmentPressureTracker.addDownloadBytesSucceeded(50); + segmentPressureTracker.addDownloadTime(101); return segmentPressureTracker; } } From f282d54dfed24fc4c2ae5fa5fd7288ce5b0075ad Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:15:09 +0530 Subject: [PATCH 09/36] Fixing API field name from routing to shards Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../stats/RemoteStoreStatsResponse.java | 2 +- .../org/opensearch/index/shard/IndexShard.java | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java index a7147e896c1c8..5b41d1710970b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponse.java @@ -93,7 +93,7 @@ public String toString() { } static final class Fields { - static final String SHARDS = "routing"; + static final String SHARDS = "shards"; static final String INDICES = "indices"; } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 6a12984f9fa27..9b9ce1cef22af 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4778,14 +4778,12 @@ private String copySegmentFiles( skippedSegments.add(file); } if (targetRemoteDirectory != null) { - targetRemoteDirectory.copyFrom(storeDirectory, file, file, IOContext.DEFAULT); - } - if (file.startsWith(IndexFileNames.SEGMENTS)) { - assert segmentNFile == null : "There should be only one SegmentInfosSnapshot file"; - segmentNFile = file; + } + } beforeSegmentDownload(downloadStatsTracker, segmentsToDownload.size(), totalSizeOfSegmentsToDownload); + // Copying segments files to local store directory from remote store directory (Shard Recovery) for (String file : segmentsToDownload) { long startTimeInMs = System.currentTimeMillis(); try { @@ -4797,6 +4795,16 @@ private String copySegmentFiles( afterSegmentDownloadFailed(downloadStatsTracker, uploadedSegments.get(file).getLength()); } } + // Copying segment files over to target remote directory from local store directory (Snapshot) + for (String file : uploadedSegments.keySet()) { + if (targetRemoteDirectory != null) { + targetRemoteDirectory.copyFrom(storeDirectory, file, file, IOContext.DEFAULT); + } + if (file.startsWith(IndexFileNames.SEGMENTS)) { + assert segmentNFile == null : "There should be only one SegmentInfosSnapshot file"; + segmentNFile = file; + } + } } finally { logger.info("Downloaded segments here: {}", downloadedSegments); logger.info("Skipped download for segments here: {}", skippedSegments); From 8de6d007a093c92bbd2a12dd9d07dcacf77893e1 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 11 Jul 2023 23:33:43 +0530 Subject: [PATCH 10/36] Changing file upload stats to sync level from file level Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/stats/RemoteStoreStats.java | 4 +- .../remote/RemoteRefreshSegmentTracker.java | 22 +++-------- .../opensearch/index/shard/IndexShard.java | 39 ++++++++++++------- .../stats/RemoteStoreStatsTestHelper.java | 6 +-- .../RemoteRefreshSegmentTrackerTests.java | 16 ++++---- 5 files changed, 41 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index c16db06e517de..ce109602cb687 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -61,7 +61,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (!shardRouting.primary()) { builder.startObject(SubFields.DOWNLOAD); builder.field(DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); - builder.startObject(DownloadStatsFields.TOTAL_FILES_DOWNLOADED) + builder.startObject(DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE) .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); @@ -224,7 +224,7 @@ static final class DownloadStatsFields { /** * Total number of segment files downloaded from the remote store for a specific shard */ - static final String TOTAL_FILES_DOWNLOADED = "total_file_downloads"; + static final String TOTAL_SYNCS_FROM_REMOTE = "total_syncs_from_remote"; /** * Total bytes of segment files downloaded from the remote store for a specific shard diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index 9b69a228d055e..d769ebf51ba63 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -196,8 +196,6 @@ public class RemoteRefreshSegmentTracker { */ private final AtomicReference downloadBytesMovingAverageReference; - private final Object downloadBytesMutex = new Object(); - /** * Provides moving average over the last N upload speed (in bytes/s) of segment files uploaded as part of remote refresh. * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. @@ -212,8 +210,6 @@ public class RemoteRefreshSegmentTracker { */ private final AtomicReference downloadBytesPerSecMovingAverageReference; - private final Object downloadBytesPerSecMutex = new Object(); - /** * Provides moving average over the last N overall upload time (in millis) as part of remote refresh.N is window size. * Wrapped with {@code AtomicReference} for dynamic changes in window size. @@ -230,8 +226,6 @@ public class RemoteRefreshSegmentTracker { private final int SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE = 20; - private final Object downloadTimeMutex = new Object(); - public RemoteRefreshSegmentTracker( ShardId shardId, int uploadBytesMovingAverageWindowSize, @@ -449,8 +443,8 @@ public long getTotalDownloadsStarted() { return totalDownloadsStarted; } - public void addTotalDownloadsStarted(long totalFiles) { - totalDownloadsStarted += totalFiles; + public void incrementTotalDownloadsStarted() { + totalDownloadsStarted += 1; } public long getTotalDownloadsFailed() { @@ -561,9 +555,7 @@ boolean isDownloadBytesAverageReady() { public void addDownloadBytes(long size) { lastSuccessfulSegmentDownloadBytes = size; - synchronized (downloadBytesMutex) { - this.downloadBytesMovingAverageReference.get().record(size); - } + this.downloadBytesMovingAverageReference.get().record(size); } boolean isUploadBytesPerSecAverageReady() { @@ -589,9 +581,7 @@ boolean isDownloadBytesPerSecAverageReady() { } public void addDownloadBytesPerSec(long bytesPerSec) { - synchronized (downloadBytesPerSecMutex) { - this.downloadBytesPerSecMovingAverageReference.get().record(bytesPerSec); - } + this.downloadBytesPerSecMovingAverageReference.get().record(bytesPerSec); } /** @@ -639,9 +629,7 @@ boolean isDownloadTimeAverageReady() { } public void addDownloadTime(long timeMs) { - synchronized (downloadTimeMutex) { - this.downloadTimeMovingAverageReference.get().record(timeMs); - } + this.downloadTimeMovingAverageReference.get().record(timeMs); } public RemoteRefreshSegmentTracker.Stats stats() { diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 9b9ce1cef22af..a4d610766752d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4777,23 +4777,32 @@ private String copySegmentFiles( } else { skippedSegments.add(file); } - if (targetRemoteDirectory != null) { - - } - } - beforeSegmentDownload(downloadStatsTracker, segmentsToDownload.size(), totalSizeOfSegmentsToDownload); - // Copying segments files to local store directory from remote store directory (Shard Recovery) - for (String file : segmentsToDownload) { + // Upload stats before starting Segment downloads + // Adding a check to prevent stats being published for NoOp runs + if (!segmentsToDownload.isEmpty()) { + beforeSegmentDownloads(downloadStatsTracker, totalSizeOfSegmentsToDownload); + } + long sizeOfDownloadedSegments = 0; + try { long startTimeInMs = System.currentTimeMillis(); - try { + // Copying segments files to local store directory from remote store directory (Shard Recovery) + for (String file : segmentsToDownload) { + // Throws IOException if download fails storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); storeDirectory.sync(Collections.singleton(file)); downloadedSegments.add(file); - afterSegmentDownloadCompleted(downloadStatsTracker, uploadedSegments.get(file).getLength(), startTimeInMs); - } catch (IOException e) { - afterSegmentDownloadFailed(downloadStatsTracker, uploadedSegments.get(file).getLength()); + sizeOfDownloadedSegments += uploadedSegments.get(file).getLength(); + } + // Upload stats after download batch is completed + // Adding a check to prevent stats being published for NoOp runs + if (!downloadedSegments.isEmpty()) { + afterSegmentDownloadsCompleted(downloadStatsTracker, sizeOfDownloadedSegments, startTimeInMs); } + } catch (IOException e) { + // Upload stats if file download batch fails + afterSegmentDownloadsFailed(downloadStatsTracker, totalSizeOfSegmentsToDownload - sizeOfDownloadedSegments); + throw e; } // Copying segment files over to target remote directory from local store directory (Snapshot) for (String file : uploadedSegments.keySet()) { @@ -4828,7 +4837,7 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo } // TODO: Move the following three methods also to the generic class with `copySegmentFiles` - private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTracker, long totalFiles, long incomingFilesSize) { + private void beforeSegmentDownloads(RemoteRefreshSegmentTracker downloadStatsTracker, long incomingFilesSize) { // The `copySegmentFiles` method is being used for both // - `syncSegmentsFromRemoteSegmentStore` (Remote store based shard recovery) // - `syncSegmentsFromGivenRemoteSegmentStore` (Snapshot inter-op with remote store) @@ -4837,11 +4846,11 @@ private void beforeSegmentDownload(RemoteRefreshSegmentTracker downloadStatsTrac if (!indexSettings.isRemoteStoreEnabled()) { return; } - downloadStatsTracker.addTotalDownloadsStarted(totalFiles); + downloadStatsTracker.incrementTotalDownloadsStarted(); downloadStatsTracker.addDownloadBytesStarted(incomingFilesSize); } - private void afterSegmentDownloadCompleted( + private void afterSegmentDownloadsCompleted( RemoteRefreshSegmentTracker downloadStatsTracker, long downloadedFileSize, long startTimeInMs @@ -4859,7 +4868,7 @@ private void afterSegmentDownloadCompleted( downloadStatsTracker.addDownloadBytesPerSec((downloadedFileSize * 1_000L) / timeTakenInMS); } - private void afterSegmentDownloadFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { + private void afterSegmentDownloadsFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { if (!indexSettings.isRemoteStoreEnabled()) { return; } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index 75696dd3a5cd2..5d5703a1ba642 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -91,19 +91,19 @@ static void compareStatsResponse( (int) pressureTrackerStats.lastDownloadTimestampMs ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILES_DOWNLOADED)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( RemoteStoreStats.SubFields.STARTED ), (int) pressureTrackerStats.totalDownloadsStarted ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILES_DOWNLOADED)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( RemoteStoreStats.SubFields.SUCCEEDED ), (int) pressureTrackerStats.totalDownloadsSucceeded ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILES_DOWNLOADED)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( RemoteStoreStats.SubFields.FAILED ), (int) pressureTrackerStats.totalDownloadsFailed diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java index bd33e69679e26..c94e8b8648528 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java @@ -290,12 +290,10 @@ public void testIncrementTotalDownloadsStarted() { pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); - long firstDownloadBatch = 20; - pressureTracker.addTotalDownloadsStarted(firstDownloadBatch); - assertEquals(firstDownloadBatch, pressureTracker.getTotalDownloadsStarted()); - long secondDownloadBatch = 20; - pressureTracker.addTotalDownloadsStarted(secondDownloadBatch); - assertEquals(firstDownloadBatch + secondDownloadBatch, pressureTracker.getTotalDownloadsStarted()); + pressureTracker.incrementTotalDownloadsStarted(); + assertEquals(1, pressureTracker.getTotalDownloadsStarted()); + pressureTracker.incrementTotalDownloadsStarted(); + assertEquals(2, pressureTracker.getTotalDownloadsStarted()); } public void testIncrementTotalDownloadsFailed() { @@ -387,9 +385,9 @@ public void testGetInflightDownloads() { pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); - pressureTracker.addTotalDownloadsStarted(1); + pressureTracker.incrementTotalDownloadsStarted(); assertEquals(1, pressureTracker.getInflightDownloads()); - pressureTracker.addTotalDownloadsStarted(1); + pressureTracker.incrementTotalDownloadsStarted(); assertEquals(2, pressureTracker.getInflightDownloads()); pressureTracker.incrementTotalDownloadsSucceeded(); assertEquals(1, pressureTracker.getInflightDownloads()); @@ -700,7 +698,7 @@ private RemoteRefreshSegmentTracker constructTracker() { segmentPressureTracker.updateRemoteRefreshTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); segmentPressureTracker.incrementRejectionCount(); segmentPressureTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); - segmentPressureTracker.addTotalDownloadsStarted(10); + segmentPressureTracker.incrementTotalDownloadsStarted(); segmentPressureTracker.incrementTotalDownloadsSucceeded(); segmentPressureTracker.addDownloadBytesStarted(50); segmentPressureTracker.addDownloadBytesSucceeded(50); From 9daeb7410a46363833986f322f3cf2484a2c83e0 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 12 Jul 2023 01:03:21 +0530 Subject: [PATCH 11/36] Retrigger integs Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> From d507ca886ad0897a0b96773de5a11e5795d486d8 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:47:55 +0530 Subject: [PATCH 12/36] Empty Commit Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> From 8c4d189f6525ebbce65020ddc9e8dec178613420 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 13 Jul 2023 17:08:22 +0530 Subject: [PATCH 13/36] Removing unused toString method Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../admin/cluster/remotestore/stats/RemoteStoreStats.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index ce109602cb687..6f368c512a137 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -9,11 +9,9 @@ package org.opensearch.action.admin.cluster.remotestore.stats; import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; @@ -127,11 +125,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } - @Override - public String toString() { - return Strings.toString(XContentType.JSON, this, true, false); - } - @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(remoteSegmentShardStats); From cdf9a49e13884ebff2186ee539371124fb4dea93 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 13 Jul 2023 17:49:01 +0530 Subject: [PATCH 14/36] Removing unused window size vars Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../index/remote/RemoteRefreshSegmentPressureSettings.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java index 2ec03af7f7a63..2a098b8f7a89b 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureSettings.java @@ -103,12 +103,6 @@ private static class Defaults { private volatile int uploadTimeMovingAverageWindowSize; - private volatile int downloadBytesMovingAverageWindowSize; - - private volatile int downloadBytesPerSecMovingAverageWindowSize; - - private volatile int downloadTimeMovingAverageWindowSize; - public RemoteRefreshSegmentPressureSettings( ClusterService clusterService, Settings settings, From b60bb40591245388f4204d176599b122b8df6b55 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Mon, 17 Jul 2023 12:12:48 +0530 Subject: [PATCH 15/36] Adding Integ tests for download stats Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 325 ++++++++++++++++-- 1 file changed, 290 insertions(+), 35 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index b76488df606cd..2befdcd320532 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -16,14 +16,21 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.test.OpenSearchIntegTestCase; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; @@ -34,6 +41,15 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; + @Override + protected Settings featureFlagSettings() { + return Settings.builder() + .put(super.featureFlagSettings()) + .put(FeatureFlags.REMOTE_STORE, "true") + .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") + .build(); + } + @Before public void setup() { setupRepo(); @@ -51,7 +67,7 @@ public void testStatsResponseFromAllNodes() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexDocs(); + indexDocs(true); // Step 2 - We find all the nodes that are present in the cluster. We make the remote store stats api call from // each of the node in the cluster and check that the response is coming as expected. @@ -73,11 +89,7 @@ public void testStatsResponseFromAllNodes() { // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well - assertAcked( - client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)) - ); - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); + changeReplicaCount(1); for (String node : nodes) { RemoteStoreStatsResponse response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); assertTrue(response.getSuccessfulShards() > 0); @@ -107,7 +119,7 @@ public void testStatsResponseAllShards() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexDocs(); + indexDocs(true); // Step 2 - We find all the nodes that are present in the cluster. We make the remote store stats api call from // each of the node in the cluster and check that the response is coming as expected. @@ -124,11 +136,7 @@ public void testStatsResponseAllShards() { // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well - assertAcked( - client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)) - ); - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); + changeReplicaCount(1); remoteStoreStatsRequestBuilder = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, null); response = remoteStoreStatsRequestBuilder.get(); assertEquals(6, response.getSuccessfulShards()); @@ -153,7 +161,7 @@ public void testStatsResponseFromLocalNode() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexDocs(); + indexDocs(true); // Step 2 - We find a data node in the cluster. We make the remote store stats api call from // each of the data node in the cluster and check that only local shards are returned. @@ -170,11 +178,7 @@ public void testStatsResponseFromLocalNode() { RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); } - assertAcked( - client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 1)) - ); - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); + changeReplicaCount(1); for (String node : nodes) { RemoteStoreStatsRequestBuilder remoteStoreStatsRequestBuilder = client(node).admin() .cluster() @@ -195,21 +199,280 @@ public void testStatsResponseFromLocalNode() { } } - private void indexDocs() { - // Indexing documents along with refreshes and flushes. - for (int i = 0; i < randomIntBetween(5, 10); i++) { - if (randomBoolean()) { - flush(INDEX_NAME); - } else { - refresh(INDEX_NAME); + public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exception { + // Scenario: + // - Create index with single primary and single replica shard + // - Disable Refresh Interval for the index + // - Index documents + // - Trigger refresh and flush + // - Assert that download stats == upload stats + // - Repeat this step for random times (between 2 and 5) + + // Create index with 1 pri and 1 replica + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); + // Disabling refresh interval + disableRefreshInterval(); + + // Index documents + for (int i = 1; i <= randomIntBetween(2, 5); i++) { + indexDocs(false); + // Running Flush & Refresh manually + flushAndRefresh(INDEX_NAME); + + // Poll for RemoteStore Stats + assertBusy(() -> { + RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; + long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; + long downloadsStarted = 0, downloadsSucceeded = 0, downloadsFailed = 0; + long downloadBytesStarted = 0, downloadBytesSucceeded = 0, downloadBytesFailed = 0; + double uploadBytesAverage = 0, downloadBytesAverage = 0; + long lastUploadedSegmentSize = 0, lastDownloadedSegmentSize = 0; + + // Iterate through the response and extract the relevant segment upload and download stats + for (RemoteStoreStats eachStatsObject : response.getRemoteStoreStats()) { + RemoteRefreshSegmentTracker.Stats stats = eachStatsObject.getStats(); + if (eachStatsObject.getShardRouting().primary()) { + uploadsStarted = stats.totalUploadsStarted; + uploadsSucceeded = stats.totalUploadsSucceeded; + uploadsFailed = stats.totalUploadsFailed; + uploadBytesStarted = stats.uploadBytesStarted; + uploadBytesSucceeded = stats.uploadBytesSucceeded; + uploadBytesFailed = stats.uploadBytesFailed; + uploadBytesAverage = stats.uploadBytesMovingAverage; + lastUploadedSegmentSize = stats.lastSuccessfulRemoteRefreshBytes; + } else { + downloadsStarted = stats.totalDownloadsStarted; + downloadsSucceeded = stats.totalDownloadsSucceeded; + downloadsFailed = stats.totalDownloadsFailed; + downloadBytesStarted = stats.downloadBytesStarted; + downloadBytesSucceeded = stats.downloadBytesSucceeded; + downloadBytesFailed = stats.downloadBytesFailed; + downloadBytesAverage = stats.downloadBytesMovingAverage; + lastDownloadedSegmentSize = stats.lastSuccessfulSegmentDownloadBytes; + } + } + // Assert Upload syncs = download syncs + assertTrue(uploadsStarted > 0 && uploadsStarted == downloadsStarted); + assertTrue(uploadsSucceeded > 0 && uploadsSucceeded == downloadsSucceeded); + assertTrue(downloadBytesStarted > 0 && uploadBytesStarted == downloadBytesStarted); + assertTrue(downloadBytesSucceeded > 0 && uploadBytesSucceeded == downloadBytesSucceeded); + // Assert zero failures + assertEquals(0, uploadsFailed); + assertEquals(0, uploadBytesFailed); + assertEquals(0, downloadsFailed); + assertEquals(0, downloadBytesFailed); + // Assert transfer bytes average is same across downloads and uploads + assertEquals(uploadBytesAverage, downloadBytesAverage, 0); + // Assert last segment size uploaded = last segment size downloaded + assertEquals(lastDownloadedSegmentSize, lastUploadedSegmentSize); + }, 30, TimeUnit.SECONDS); + } + } + + public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() throws Exception { + // Scenario: + // - Create index with single primary and N-1 replica shards (N = no of data nodes) + // - Disable Refresh Interval for the index + // - Index documents + // - Trigger refresh and flush + // - Assert that download stats == upload stats + // - Repeat this step for random times (between 2 and 5) + + // Crete index + int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); + createIndex(INDEX_NAME, remoteStoreIndexSettings(dataNodeCount - 1, 1)); + + // Disable refresh interval + disableRefreshInterval(); + ensureGreen(INDEX_NAME); + + for (int i = 0; i < randomIntBetween(2, 5); i++) { + indexDocs(false); + + // Running Flush & Refresh manually + flushAndRefresh(INDEX_NAME); + + assertBusy(() -> { + RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + assertEquals(3, response.getSuccessfulShards()); + long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; + long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; + double uploadBytesAverage = 0; + List downloadsStarted = new ArrayList<>(), downloadsSucceeded = new ArrayList<>(), downloadsFailed = + new ArrayList<>(); + List downloadBytesStarted = new ArrayList<>(), downloadBytesSucceeded = new ArrayList<>(), downloadBytesFailed = + new ArrayList<>(); + List downloadBytesAverage = new ArrayList<>(); + long lastUploadedSegmentSize = 0; + List lastDownloadedSegmentSize = new ArrayList<>(); + + // Assert that stats for primary shard and replica shard set are equal + for (RemoteStoreStats eachStatsObject : response.getRemoteStoreStats()) { + RemoteRefreshSegmentTracker.Stats stats = eachStatsObject.getStats(); + if (eachStatsObject.getShardRouting().primary()) { + uploadsStarted = stats.totalUploadsStarted; + uploadsSucceeded = stats.totalUploadsSucceeded; + uploadsFailed = stats.totalUploadsFailed; + uploadBytesStarted = stats.uploadBytesStarted; + uploadBytesSucceeded = stats.uploadBytesSucceeded; + uploadBytesFailed = stats.uploadBytesFailed; + uploadBytesAverage = stats.uploadBytesMovingAverage; + lastUploadedSegmentSize = stats.lastSuccessfulRemoteRefreshBytes; + } else { + downloadsStarted.add(stats.totalDownloadsStarted); + downloadsSucceeded.add(stats.totalDownloadsSucceeded); + downloadsFailed.add(stats.totalDownloadsFailed); + downloadBytesStarted.add(stats.downloadBytesStarted); + downloadBytesSucceeded.add(stats.downloadBytesSucceeded); + downloadBytesFailed.add(stats.downloadBytesFailed); + downloadBytesAverage.add(stats.downloadBytesMovingAverage); + lastDownloadedSegmentSize.add(stats.lastSuccessfulSegmentDownloadBytes); + } + } + + assertEquals(0, uploadsFailed); + assertEquals(0, uploadBytesFailed); + for (int j = 0; j < response.getSuccessfulShards() - 1; j++) { + assertEquals(uploadsStarted, (long) downloadsStarted.get(j)); + assertEquals(uploadsSucceeded, (long) downloadsSucceeded.get(j)); + assertEquals(0, (long) downloadsFailed.get(j)); + assertEquals(uploadBytesStarted, (long) downloadBytesStarted.get(j)); + assertEquals(uploadBytesSucceeded, (long) downloadBytesSucceeded.get(j)); + assertEquals(0, (long) downloadBytesFailed.get(j)); + assertEquals(uploadBytesAverage, downloadBytesAverage.get(j), 0); + assertEquals(lastUploadedSegmentSize, (long) lastDownloadedSegmentSize.get(j)); + } + }, 45, TimeUnit.SECONDS); + } + } + + public void testStatsOnShardRelocation() { + // Scenario: + // - Create index with single primary and single replica shard + // - Index documents + // - Reroute replica shard to one of the remaining nodes + // - Assert that remote store stats reflects the new node ID + + // Create index + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); + // Index docs + indexDocs(true); + + // Fetch current set of nodes in the cluster + List currentNodesInCluster = getClusterState().nodes() + .getDataNodes() + .values() + .stream() + .map(DiscoveryNode::getId) + .collect(Collectors.toList()); + DiscoveryNode[] discoveryNodesForIndex = client().admin().cluster().prepareSearchShards(INDEX_NAME).get().getNodes(); + + // Fetch nodes with shard copies of the created index + List nodeIdsWithShardCopies = new ArrayList<>(); + Arrays.stream(discoveryNodesForIndex).forEach(eachNode -> nodeIdsWithShardCopies.add(eachNode.getId())); + + // Fetch nodes which does not have any copies of the index + List nodesIdsWithoutShardCopy = currentNodesInCluster.stream() + .filter(eachNode -> !nodeIdsWithShardCopies.contains(eachNode)) + .collect(Collectors.toList()); + assertEquals(1, nodesIdsWithoutShardCopy.size()); + + // Manually reroute shard to a ndoe which does not have any shard copy at present + ShardRouting replicaShardRouting = getClusterState().routingTable() + .index(INDEX_NAME) + .shard(0) + .assignedShards() + .stream() + .filter(shard -> !shard.primary()) + .collect(Collectors.toList()) + .get(0); + String sourceNode = replicaShardRouting.currentNodeId(); + String destinationNode = nodesIdsWithoutShardCopy.get(0); + relocateShard(0, sourceNode, destinationNode); + RemoteStoreStats[] allShardsStats = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get().getRemoteStoreStats(); + RemoteStoreStats replicaShardStat = Arrays.stream(allShardsStats) + .filter(eachStat -> !eachStat.getShardRouting().primary()) + .collect(Collectors.toList()) + .get(0); + + // Assert that remote store stats reflect the new shard state + assertEquals(ShardRoutingState.STARTED, replicaShardStat.getShardRouting().state()); + assertEquals(destinationNode, replicaShardStat.getShardRouting().currentNodeId()); + } + + public void testStatsOnShardUnassigned() throws IOException { + // Scenario: + // - Create index with single primary and two replica shard + // - Index documents + // - Stop one data node + // - Assert: + // a. Total shard Count in the response object is equal to the previous node count + // b. Successful shard count in the response object is equal to the new node count + createIndex(INDEX_NAME, remoteStoreIndexSettings(2, 1)); + indexDocs(true); + int dataNodeCountBeforeStop = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); + internalCluster().stopRandomDataNode(); + RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + int dataNodeCountAfterStop = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); + assertEquals(dataNodeCountBeforeStop, response.getTotalShards()); + assertEquals(dataNodeCountAfterStop, response.getSuccessfulShards()); + } + + private void indexDocs(boolean withFlushAndRefresh) { + if (withFlushAndRefresh) { + // Indexing documents along with refreshes and flushes. + for (int i = 0; i < randomIntBetween(5, 10); i++) { + if (randomBoolean()) { + flush(INDEX_NAME); + } else { + refresh(INDEX_NAME); + } + int numberOfOperations = randomIntBetween(20, 50); + for (int j = 0; j < numberOfOperations; j++) { + indexSingleDoc(); + } } - int numberOfOperations = randomIntBetween(20, 50); - for (int j = 0; j < numberOfOperations; j++) { + } else { + for (int j = 0; j < randomIntBetween(5, 10); j++) { indexSingleDoc(); } } } + private IndexResponse indexSingleDoc() { + return client().prepareIndex(INDEX_NAME) + .setId(UUIDs.randomBase64UUID()) + .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) + .get(); + } + + private void changeReplicaCount(int replicaCount) { + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, replicaCount)) + ); + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + } + + private void relocateShard(int shardId, String sourceNode, String destNode) { + assertAcked(client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, shardId, sourceNode, destNode))); + ensureGreen(INDEX_NAME); + } + + private void disableRefreshInterval() { + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)) + ); + ensureGreen(INDEX_NAME); + } + private void validateUploadStats(RemoteRefreshSegmentTracker.Stats stats) { assertEquals(0, stats.refreshTimeLagMs); assertEquals(stats.localRefreshNumber, stats.remoteRefreshNumber); @@ -240,12 +503,4 @@ private void validateDownloadStats(RemoteRefreshSegmentTracker.Stats stats) { assertTrue(stats.downloadBytesPerSecMovingAverage > 0); assertTrue(stats.downloadTimeMovingAverage > 0); } - - private IndexResponse indexSingleDoc() { - return client().prepareIndex(INDEX_NAME) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - } From c62926fba398b5b261b273c148861c2343216832 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 18 Jul 2023 15:47:18 +0530 Subject: [PATCH 16/36] Addressing comments Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 108 +++++++++-- .../remotestore/stats/RemoteStoreStats.java | 44 ++--- .../remote/RemoteRefreshSegmentTracker.java | 41 ++-- .../opensearch/index/shard/IndexShard.java | 78 +++----- .../stats/RemoteStoreStatsResponseTests.java | 93 ++++++++- .../stats/RemoteStoreStatsTestHelper.java | 176 +++++++++++++----- .../stats/RemoteStoreStatsTests.java | 154 +++++++++++---- 7 files changed, 490 insertions(+), 204 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 2befdcd320532..2b3e2ce4ec84d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -9,10 +9,12 @@ package org.opensearch.remotestore; import org.junit.Before; +import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsRequestBuilder; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; @@ -32,6 +34,7 @@ import java.util.Locale; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -43,11 +46,7 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { @Override protected Settings featureFlagSettings() { - return Settings.builder() - .put(super.featureFlagSettings()) - .put(FeatureFlags.REMOTE_STORE, "true") - .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") - .build(); + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build(); } @Before @@ -76,6 +75,7 @@ public void testStatsResponseFromAllNodes() { String shardId = "0"; for (String node : nodes) { RemoteStoreStatsResponse response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); + logger.info("Stats Response : " + response); assertTrue(response.getSuccessfulShards() > 0); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); @@ -85,11 +85,12 @@ public void testStatsResponseFromAllNodes() { assertEquals(1, matches.size()); RemoteRefreshSegmentTracker.Stats stats = matches.get(0).getStats(); validateUploadStats(stats); + assertEquals(0, stats.totalDownloadsStarted); } // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well - changeReplicaCount(1); + changeReplicaCountAndEnsureGreen(1); for (String node : nodes) { RemoteStoreStatsResponse response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); assertTrue(response.getSuccessfulShards() > 0); @@ -101,11 +102,14 @@ public void testStatsResponseFromAllNodes() { assertEquals(2, matches.size()); for (RemoteStoreStats stat : matches) { ShardRouting routing = stat.getShardRouting(); + validateShardRouting(routing); RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); + assertEquals(0, stats.totalDownloadsStarted); } else { validateDownloadStats(stats); + assertEquals(0, stats.totalUploadsStarted); } } } @@ -133,21 +137,24 @@ public void testStatsResponseAllShards() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 3); RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); + assertEquals(0, stats.totalDownloadsStarted); // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well - changeReplicaCount(1); - remoteStoreStatsRequestBuilder = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, null); - response = remoteStoreStatsRequestBuilder.get(); + changeReplicaCountAndEnsureGreen(1); + response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, null).get(); assertEquals(6, response.getSuccessfulShards()); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 6); for (RemoteStoreStats stat : response.getRemoteStoreStats()) { ShardRouting routing = stat.getShardRouting(); + validateShardRouting(routing); stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); + assertEquals(0, stats.totalDownloadsStarted); } else { validateDownloadStats(stats); + assertEquals(0, stats.totalUploadsStarted); } } @@ -177,8 +184,9 @@ public void testStatsResponseFromLocalNode() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 1); RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); + assertEquals(0, stats.totalDownloadsStarted); } - changeReplicaCount(1); + changeReplicaCountAndEnsureGreen(1); for (String node : nodes) { RemoteStoreStatsRequestBuilder remoteStoreStatsRequestBuilder = client(node).admin() .cluster() @@ -189,11 +197,14 @@ public void testStatsResponseFromLocalNode() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); for (RemoteStoreStats stat : response.getRemoteStoreStats()) { ShardRouting routing = stat.getShardRouting(); + validateShardRouting(routing); RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); + assertEquals(0, stats.totalDownloadsStarted); } else { validateDownloadStats(stats); + assertEquals(0, stats.totalUploadsStarted); } } } @@ -286,7 +297,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr // Disable refresh interval disableRefreshInterval(); ensureGreen(INDEX_NAME); - + int currentNodesInCluster = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); for (int i = 0; i < randomIntBetween(2, 5); i++) { indexDocs(false); @@ -295,7 +306,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr assertBusy(() -> { RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - assertEquals(3, response.getSuccessfulShards()); + assertEquals(currentNodesInCluster, response.getSuccessfulShards()); long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; double uploadBytesAverage = 0; @@ -373,12 +384,12 @@ public void testStatsOnShardRelocation() { Arrays.stream(discoveryNodesForIndex).forEach(eachNode -> nodeIdsWithShardCopies.add(eachNode.getId())); // Fetch nodes which does not have any copies of the index - List nodesIdsWithoutShardCopy = currentNodesInCluster.stream() + List nodeIdsWithoutShardCopy = currentNodesInCluster.stream() .filter(eachNode -> !nodeIdsWithShardCopies.contains(eachNode)) .collect(Collectors.toList()); - assertEquals(1, nodesIdsWithoutShardCopy.size()); + assertEquals(1, nodeIdsWithoutShardCopy.size()); - // Manually reroute shard to a ndoe which does not have any shard copy at present + // Manually reroute shard to a node which does not have any shard copy at present ShardRouting replicaShardRouting = getClusterState().routingTable() .index(INDEX_NAME) .shard(0) @@ -388,7 +399,7 @@ public void testStatsOnShardRelocation() { .collect(Collectors.toList()) .get(0); String sourceNode = replicaShardRouting.currentNodeId(); - String destinationNode = nodesIdsWithoutShardCopy.get(0); + String destinationNode = nodeIdsWithoutShardCopy.get(0); relocateShard(0, sourceNode, destinationNode); RemoteStoreStats[] allShardsStats = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get().getRemoteStoreStats(); RemoteStoreStats replicaShardStat = Arrays.stream(allShardsStats) @@ -419,6 +430,50 @@ public void testStatsOnShardUnassigned() throws IOException { assertEquals(dataNodeCountAfterStop, response.getSuccessfulShards()); } + public void testStatsOnRemoteStoreRestore() throws IOException { + // Creating an index with primary shard count == total nodes in cluster and 0 replicas + int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); + createIndex(INDEX_NAME, remoteStoreIndexSettings(0, dataNodeCount)); + + // Index some docs to ensure segments being uploaded to remote store + indexDocs(true); + + // Stop one data node to force the index into a red state + internalCluster().stopRandomDataNode(); + ensureRed(INDEX_NAME); + + // Start another data node to fulfil the previously launched capacity + internalCluster().startDataOnlyNode(); + + // Restore index from remote store + assertAcked(client().admin().indices().prepareClose(INDEX_NAME).get()); + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + + // Ensure that the index is green + ensureGreen(INDEX_NAME); + + // Index some more docs to force segment uploads to remote store + indexDocs(true); + + RemoteStoreStats[] remoteStoreStats = client().admin() + .cluster() + .prepareRemoteStoreStats(INDEX_NAME, "0") + .get() + .getRemoteStoreStats(); + Arrays.stream(remoteStoreStats).forEach(statObject -> { + RemoteRefreshSegmentTracker.Stats segmentTracker = statObject.getStats(); + // Assert that we have both upload and download stats for the index + assertTrue( + segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0 + ); + assertTrue( + segmentTracker.totalDownloadsStarted > 0 + && segmentTracker.totalDownloadsSucceeded > 0 + && segmentTracker.totalDownloadsFailed == 0 + ); + }); + } + private void indexDocs(boolean withFlushAndRefresh) { if (withFlushAndRefresh) { // Indexing documents along with refreshes and flushes. @@ -447,7 +502,7 @@ private IndexResponse indexSingleDoc() { .get(); } - private void changeReplicaCount(int replicaCount) { + private void changeReplicaCountAndEnsureGreen(int replicaCount) { assertAcked( client().admin() .indices() @@ -470,7 +525,6 @@ private void disableRefreshInterval() { .prepareUpdateSettings(INDEX_NAME) .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)) ); - ensureGreen(INDEX_NAME); } private void validateUploadStats(RemoteRefreshSegmentTracker.Stats stats) { @@ -503,4 +557,22 @@ private void validateDownloadStats(RemoteRefreshSegmentTracker.Stats stats) { assertTrue(stats.downloadBytesPerSecMovingAverage > 0); assertTrue(stats.downloadTimeMovingAverage > 0); } + + // Validate if the shardRouting obtained from cluster state contains the exact same routing object + // parameters as obtained from the remote store stats API + private void validateShardRouting(ShardRouting routing) { + Stream currentRoutingTable = getClusterState().routingTable() + .getIndicesRouting() + .get(INDEX_NAME) + .shard(routing.id()) + .assignedShards() + .stream(); + assertTrue( + currentRoutingTable.anyMatch( + r -> (r.currentNodeId().equals(routing.currentNodeId()) + && r.state().equals(routing.state()) + && r.primary() == routing.primary()) + ) + ); + } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index b83c2bfdda39c..9c732ce2c3161 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -56,15 +56,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(RoutingFields.NODE_ID, shardRouting.currentNodeId()); builder.endObject(); builder.startObject(Fields.SEGMENT); - if (!shardRouting.primary()) { - builder.startObject(SubFields.DOWNLOAD); - builder.field(DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); + builder.startObject(SubFields.DOWNLOAD); + // Ensuring that we are not showing 0 metrics to the user + if (remoteSegmentShardStats.totalDownloadsStarted != 0) { + builder.field(DownloadStatsFields.LAST_SYNC_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); builder.startObject(DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE) .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); builder.endObject(); - builder.startObject(DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES) + builder.startObject(DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES) .field(SubFields.STARTED, remoteSegmentShardStats.downloadBytesStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.downloadBytesSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.downloadBytesFailed); @@ -79,14 +80,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS) .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadTimeMovingAverage); builder.endObject(); - builder.endObject(); - } else { - builder.startObject(SubFields.DOWNLOAD); - builder.endObject(); } - if (shardRouting.primary()) { - builder.startObject(SubFields.UPLOAD) - .field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) + builder.endObject(); + builder.startObject(SubFields.UPLOAD); + // Ensuring that we are not showing 0 metrics to the user + if (remoteSegmentShardStats.totalUploadsStarted != 0) { + builder.field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) .field(UploadStatsFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) .field(UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) .field( @@ -96,7 +95,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field(UploadStatsFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) .field(UploadStatsFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) .field(UploadStatsFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); - builder.startObject(UploadStatsFields.TOTAL_REMOTE_REFRESH) + builder.startObject(UploadStatsFields.TOTAL_SYNCS_TO_REMOTE) .field(SubFields.STARTED, remoteSegmentShardStats.totalUploadsStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalUploadsSucceeded) .field(SubFields.FAILED, remoteSegmentShardStats.totalUploadsFailed); @@ -116,12 +115,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS) .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadTimeMovingAverage); builder.endObject(); - builder.endObject(); - } else { - builder.startObject(SubFields.UPLOAD); - builder.endObject(); } builder.endObject(); + builder.endObject(); return builder.endObject(); } @@ -185,7 +181,7 @@ static final class UploadStatsFields { /** * Represents the number of remote refreshes */ - static final String TOTAL_REMOTE_REFRESH = "total_remote_refresh"; + static final String TOTAL_SYNCS_TO_REMOTE = "total_syncs_to_remote"; /** * Represents the total uploads to remote store in bytes @@ -210,19 +206,19 @@ static final class UploadStatsFields { static final class DownloadStatsFields { /** - * Last successful segment download timestamp in milliseconds + * Last successful sync from remote in milliseconds */ - static final String LAST_DOWNLOAD_TIMESTAMP = "last_download_timestamp"; + static final String LAST_SYNC_TIMESTAMP = "last_sync_timestamp"; /** - * Total number of segment files downloaded from the remote store for a specific shard + * Total number of sync from the remote store for a specific shard */ static final String TOTAL_SYNCS_FROM_REMOTE = "total_syncs_from_remote"; /** * Total bytes of segment files downloaded from the remote store for a specific shard */ - static final String TOTAL_FILE_DOWNLOADS_IN_BYTES = "total_file_downloads_in_bytes"; + static final String TOTAL_DOWNLOADS_IN_BYTES = "total_downloads_in_bytes"; /** * Size of each segment file downloaded from the remote store @@ -241,7 +237,7 @@ static final class DownloadStatsFields { } /** - * Reusable sub fields for {@link Fields} + * Reusable sub fields for {@link UploadStatsFields} and {@link DownloadStatsFields} */ static final class SubFields { static final String STARTED = "started"; @@ -252,12 +248,12 @@ static final class SubFields { static final String UPLOAD = "upload"; /** - * Moving avg over last N values stat for a {@link Fields} + * Moving avg over last N values stat */ static final String MOVING_AVG = "moving_avg"; /** - * Most recent successful attempt stat for a {@link Fields} + * Most recent successful attempt stat */ static final String LAST_SUCCESSFUL = "last_successful"; } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java index c036208a05098..72342927d9316 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java @@ -192,9 +192,9 @@ public class RemoteRefreshSegmentTracker { /** * Provides moving average over the last N total size in bytes of segment files downloaded from the remote store. - * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. + * N is window size */ - private final AtomicReference downloadBytesMovingAverageReference; + private volatile MovingAverage downloadBytesMovingAverageReference; /** * Provides moving average over the last N upload speed (in bytes/s) of segment files uploaded as part of remote refresh. @@ -206,9 +206,9 @@ public class RemoteRefreshSegmentTracker { /** * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the remote store. - * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. + * N is window size */ - private final AtomicReference downloadBytesPerSecMovingAverageReference; + private volatile MovingAverage downloadBytesPerSecMovingAverageReference; /** * Provides moving average over the last N overall upload time (in millis) as part of remote refresh.N is window size. @@ -220,9 +220,8 @@ public class RemoteRefreshSegmentTracker { /** * Provides moving average over the last N overall download time (in millis) of segments downloaded from the remote store. - * Wrapped with {@code AtomicReference} for dynamic changes in window size. */ - private final AtomicReference downloadTimeMovingAverageReference; + private volatile MovingAverage downloadTimeMovingAverageReference; private final int SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE = 20; @@ -243,9 +242,9 @@ public RemoteRefreshSegmentTracker( uploadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesMovingAverageWindowSize)); uploadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesPerSecMovingAverageWindowSize)); uploadTimeMsMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadTimeMsMovingAverageWindowSize)); - downloadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE)); - downloadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE)); - downloadTimeMovingAverageReference = new AtomicReference<>(new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE)); + downloadBytesMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); + downloadBytesPerSecMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); + downloadTimeMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); latestLocalFileNameLengthMap = new HashMap<>(); } @@ -546,16 +545,16 @@ void updateUploadBytesMovingAverageWindowSize(int updatedSize) { } boolean isDownloadBytesAverageReady() { - return downloadBytesMovingAverageReference.get().isReady(); + return downloadBytesMovingAverageReference.isReady(); } double getDownloadBytesAverage() { - return downloadBytesMovingAverageReference.get().getAverage(); + return downloadBytesMovingAverageReference.getAverage(); } public void addDownloadBytes(long size) { lastSuccessfulSegmentDownloadBytes = size; - this.downloadBytesMovingAverageReference.get().record(size); + this.downloadBytesMovingAverageReference.record(size); } boolean isUploadBytesPerSecAverageReady() { @@ -573,15 +572,15 @@ public void addUploadBytesPerSec(long bytesPerSec) { } boolean isDownloadBytesPerSecAverageReady() { - return downloadBytesPerSecMovingAverageReference.get().isReady(); + return downloadBytesPerSecMovingAverageReference.isReady(); } double getDownloadBytesPerSecAverage() { - return downloadBytesPerSecMovingAverageReference.get().getAverage(); + return downloadBytesPerSecMovingAverageReference.getAverage(); } public void addDownloadBytesPerSec(long bytesPerSec) { - this.downloadBytesPerSecMovingAverageReference.get().record(bytesPerSec); + this.downloadBytesPerSecMovingAverageReference.record(bytesPerSec); } /** @@ -621,15 +620,15 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { } boolean isDownloadTimeAverageReady() { - return downloadTimeMovingAverageReference.get().isReady(); + return downloadTimeMovingAverageReference.isReady(); } double getDownloadTimeAverage() { - return downloadTimeMovingAverageReference.get().getAverage(); + return downloadTimeMovingAverageReference.getAverage(); } public void addDownloadTime(long timeMs) { - this.downloadTimeMovingAverageReference.get().record(timeMs); + this.downloadTimeMovingAverageReference.record(timeMs); } public RemoteRefreshSegmentTracker.Stats stats() { @@ -660,9 +659,9 @@ public RemoteRefreshSegmentTracker.Stats stats() { uploadBytesPerSecMovingAverageReference.get().getAverage(), uploadTimeMsMovingAverageReference.get().getAverage(), lastSuccessfulSegmentDownloadBytes, - downloadBytesMovingAverageReference.get().getAverage(), - downloadBytesPerSecMovingAverageReference.get().getAverage(), - downloadTimeMovingAverageReference.get().getAverage(), + downloadBytesMovingAverageReference.getAverage(), + downloadBytesPerSecMovingAverageReference.getAverage(), + downloadTimeMovingAverageReference.getAverage(), getBytesLag() ); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 89b08bd1ecf43..e4d99aded0268 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4780,7 +4780,6 @@ private String copySegmentFiles( List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); List segmentsToDownload = new ArrayList<>(); - long totalSizeOfSegmentsToDownload = 0; String segmentNFile = null; try { Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); @@ -4793,35 +4792,46 @@ private String copySegmentFiles( long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { segmentsToDownload.add(file); - totalSizeOfSegmentsToDownload += uploadedSegments.get(file).getLength(); + // Adding size of the file to attempted download stats + if (indexSettings.isRemoteStoreEnabled()) { + downloadStatsTracker.addDownloadBytesStarted(uploadedSegments.get(file).getLength()); + } } else { skippedSegments.add(file); } } // Upload stats before starting Segment downloads // Adding a check to prevent stats being published for NoOp runs - if (!segmentsToDownload.isEmpty()) { - beforeSegmentDownloads(downloadStatsTracker, totalSizeOfSegmentsToDownload); + if (!segmentsToDownload.isEmpty() && indexSettings.isRemoteStoreEnabled()) { + downloadStatsTracker.incrementTotalDownloadsStarted(); } long sizeOfDownloadedSegments = 0; - try { - long startTimeInMs = System.currentTimeMillis(); - // Copying segments files to local store directory from remote store directory (Shard Recovery) - for (String file : segmentsToDownload) { + long startTimeInMs = System.currentTimeMillis(); + // Copying segments files to local store directory from remote store directory (Shard Recovery) + for (String file : segmentsToDownload) { + long fileSize = uploadedSegments.get(file).getLength(); + try { // Throws IOException if download fails storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); downloadedSegments.add(file); - sizeOfDownloadedSegments += uploadedSegments.get(file).getLength(); - } - // Upload stats after download batch is completed - // Adding a check to prevent stats being published for NoOp runs - if (!downloadedSegments.isEmpty()) { - afterSegmentDownloadsCompleted(downloadStatsTracker, sizeOfDownloadedSegments, startTimeInMs); + if (indexSettings.isRemoteStoreEnabled()) { + sizeOfDownloadedSegments += fileSize; + // Adding size of the file to completed download stats + downloadStatsTracker.addDownloadBytesSucceeded(fileSize); + } + } catch (IOException e) { + if (indexSettings.isRemoteStoreEnabled()) { + // Adding size of the file to failed download stats + downloadStatsTracker.addDownloadBytesFailed(fileSize); + // Incrementing overall remote sync failure counter + downloadStatsTracker.incrementTotalDownloadsFailed(); + } + throw e; } - } catch (IOException e) { - // Upload stats if file download batch fails - afterSegmentDownloadsFailed(downloadStatsTracker, totalSizeOfSegmentsToDownload - sizeOfDownloadedSegments); - throw e; + } + // Upload stats after download batch is completed + if (!downloadedSegments.isEmpty() && indexSettings.isRemoteStoreEnabled()) { + afterSyncCompleted(downloadStatsTracker, sizeOfDownloadedSegments, startTimeInMs); } // Copying segment files over to target remote directory from local store directory (Snapshot) for (String file : uploadedSegments.keySet()) { @@ -4855,46 +4865,16 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo return false; } - // TODO: Move the following three methods also to the generic class with `copySegmentFiles` - private void beforeSegmentDownloads(RemoteRefreshSegmentTracker downloadStatsTracker, long incomingFilesSize) { - // The `copySegmentFiles` method is being used for both - // - `syncSegmentsFromRemoteSegmentStore` (Remote store based shard recovery) - // - `syncSegmentsFromGivenRemoteSegmentStore` (Snapshot inter-op with remote store) - // Since there is a provision to disable remote store in the index that is being restored from a snapshot, - // adding this check to ensure that the stats code path does not get executed - if (!indexSettings.isRemoteStoreEnabled()) { - return; - } - downloadStatsTracker.incrementTotalDownloadsStarted(); - downloadStatsTracker.addDownloadBytesStarted(incomingFilesSize); - } - - private void afterSegmentDownloadsCompleted( - RemoteRefreshSegmentTracker downloadStatsTracker, - long downloadedFileSize, - long startTimeInMs - ) { - if (!indexSettings.isRemoteStoreEnabled()) { - return; - } + private void afterSyncCompleted(RemoteRefreshSegmentTracker downloadStatsTracker, long downloadedFileSize, long startTimeInMs) { long currentTimeInMs = System.currentTimeMillis(); downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); downloadStatsTracker.incrementTotalDownloadsSucceeded(); downloadStatsTracker.addDownloadBytes(downloadedFileSize); - downloadStatsTracker.addDownloadBytesSucceeded(downloadedFileSize); long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); downloadStatsTracker.addDownloadTime(timeTakenInMS); downloadStatsTracker.addDownloadBytesPerSec((downloadedFileSize * 1_000L) / timeTakenInMS); } - private void afterSegmentDownloadsFailed(RemoteRefreshSegmentTracker downloadStatsTracker, long failedFileSize) { - if (!indexSettings.isRemoteStoreEnabled()) { - return; - } - downloadStatsTracker.incrementTotalDownloadsFailed(); - downloadStatsTracker.addDownloadBytesFailed(failedFileSize); - } - /** * Returns the maximum sequence number of either update or delete operations have been processed in this shard * or the sequence number from {@link #advanceMaxSeqNoOfUpdatesOrDeletes(long)}. An index request is considered diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java index d83ead831d229..1a562e68f1745 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java @@ -24,8 +24,10 @@ import java.util.Map; import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.compareStatsResponse; -import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createPressureTrackerStats; import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createShardRouting; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createStatsForNewPrimary; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createStatsForNewReplica; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createStatsForRemoteStoreRestoredPrimary; import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; public class RemoteStoreStatsResponseTests extends OpenSearchTestCase { @@ -45,10 +47,10 @@ public void tearDown() throws Exception { threadPool.shutdownNow(); } - public void testSerialization() throws Exception { - RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); + public void testSerializationForPrimary() throws Exception { + RemoteRefreshSegmentTracker.Stats mockPrimaryTrackerStats = createStatsForNewPrimary(shardId); ShardRouting primaryShardRouting = createShardRouting(shardId, true); - RemoteStoreStats primaryShardStats = new RemoteStoreStats(pressureTrackerStats, primaryShardRouting); + RemoteStoreStats primaryShardStats = new RemoteStoreStats(mockPrimaryTrackerStats, primaryShardRouting); RemoteStoreStatsResponse statsResponse = new RemoteStoreStatsResponse( new RemoteStoreStats[] { primaryShardStats }, 1, @@ -70,6 +72,87 @@ public void testSerialization() throws Exception { Map shardsObject = (Map) ((Map) indicesObject.get("index")).get("shards"); ArrayList> perShardNumberObject = (ArrayList>) shardsObject.get("0"); assertEquals(perShardNumberObject.size(), 1); - compareStatsResponse(perShardNumberObject.get(0), pressureTrackerStats, primaryShardRouting); + Map perShardCopyObject = perShardNumberObject.get(0); + compareStatsResponse(perShardCopyObject, mockPrimaryTrackerStats, primaryShardRouting); + } + + public void testSerializationForBothPrimaryAndReplica() throws Exception { + RemoteRefreshSegmentTracker.Stats mockPrimaryTrackerStats = createStatsForNewPrimary(shardId); + RemoteRefreshSegmentTracker.Stats mockReplicaTrackerStats = createStatsForNewReplica(shardId); + ShardRouting primaryShardRouting = createShardRouting(shardId, true); + ShardRouting replicaShardRouting = createShardRouting(shardId, false); + RemoteStoreStats primaryShardStats = new RemoteStoreStats(mockPrimaryTrackerStats, primaryShardRouting); + RemoteStoreStats replicaShardStats = new RemoteStoreStats(mockReplicaTrackerStats, replicaShardRouting); + RemoteStoreStatsResponse statsResponse = new RemoteStoreStatsResponse( + new RemoteStoreStats[] { primaryShardStats, replicaShardStats }, + 2, + 2, + 0, + new ArrayList() + ); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + statsResponse.toXContent(builder, EMPTY_PARAMS); + Map jsonResponseObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()) + .v2(); + Map metadataShardsObject = (Map) jsonResponseObject.get("_shards"); + assertEquals(2, metadataShardsObject.get("total")); + assertEquals(2, metadataShardsObject.get("successful")); + assertEquals(0, metadataShardsObject.get("failed")); + Map indicesObject = (Map) jsonResponseObject.get("indices"); + assertTrue(indicesObject.containsKey("index")); + Map shardsObject = (Map) ((Map) indicesObject.get("index")).get("shards"); + ArrayList> perShardNumberObject = (ArrayList>) shardsObject.get("0"); + assertEquals(2, perShardNumberObject.size()); + perShardNumberObject.forEach(shardObject -> { + boolean isPrimary = (boolean) ((Map) shardObject.get(RemoteStoreStats.Fields.ROUTING)).get( + RemoteStoreStats.RoutingFields.PRIMARY + ); + if (isPrimary) { + compareStatsResponse(shardObject, mockPrimaryTrackerStats, primaryShardRouting); + } else { + compareStatsResponse(shardObject, mockReplicaTrackerStats, replicaShardRouting); + } + }); + } + + public void testSerializationForBothRemoteStoreRestoredPrimaryAndReplica() throws Exception { + RemoteRefreshSegmentTracker.Stats mockPrimaryTrackerStats = createStatsForRemoteStoreRestoredPrimary(shardId); + RemoteRefreshSegmentTracker.Stats mockReplicaTrackerStats = createStatsForNewReplica(shardId); + ShardRouting primaryShardRouting = createShardRouting(shardId, true); + ShardRouting replicaShardRouting = createShardRouting(shardId, false); + RemoteStoreStats primaryShardStats = new RemoteStoreStats(mockPrimaryTrackerStats, primaryShardRouting); + RemoteStoreStats replicaShardStats = new RemoteStoreStats(mockReplicaTrackerStats, replicaShardRouting); + RemoteStoreStatsResponse statsResponse = new RemoteStoreStatsResponse( + new RemoteStoreStats[] { primaryShardStats, replicaShardStats }, + 2, + 2, + 0, + new ArrayList() + ); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + statsResponse.toXContent(builder, EMPTY_PARAMS); + Map jsonResponseObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()) + .v2(); + Map metadataShardsObject = (Map) jsonResponseObject.get("_shards"); + assertEquals(2, metadataShardsObject.get("total")); + assertEquals(2, metadataShardsObject.get("successful")); + assertEquals(0, metadataShardsObject.get("failed")); + Map indicesObject = (Map) jsonResponseObject.get("indices"); + assertTrue(indicesObject.containsKey("index")); + Map shardsObject = (Map) ((Map) indicesObject.get("index")).get("shards"); + ArrayList> perShardNumberObject = (ArrayList>) shardsObject.get("0"); + assertEquals(2, perShardNumberObject.size()); + perShardNumberObject.forEach(shardObject -> { + boolean isPrimary = (boolean) ((Map) shardObject.get(RemoteStoreStats.Fields.ROUTING)).get( + RemoteStoreStats.RoutingFields.PRIMARY + ); + if (isPrimary) { + compareStatsResponse(shardObject, mockPrimaryTrackerStats, primaryShardRouting); + } else { + compareStatsResponse(shardObject, mockReplicaTrackerStats, replicaShardRouting); + } + }); } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index b076e6ed14031..c6cef9357dc23 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -24,48 +24,118 @@ * Helper utilities for Remote Store stats tests */ public class RemoteStoreStatsTestHelper { - static RemoteRefreshSegmentTracker.Stats createPressureTrackerStats(ShardId shardId) { + static RemoteRefreshSegmentTracker.Stats createStatsForNewPrimary(ShardId shardId) { return new RemoteRefreshSegmentTracker.Stats( shardId, 101, 102, 100, - 3, + 0, 10, 2, 10, 5, 5, - 10, - 10, - 5, - 5, - 5, - 5, + 0, + 0, + 0, 5, 5, 5, + 0, + 0, + 0, 3, 2, 5, 2, 3, 4, - 9, - 3, - 4, - 9, + 0, + 0, + 0, + 0, 5 ); } + static RemoteRefreshSegmentTracker.Stats createStatsForNewReplica(ShardId shardId) { + return new RemoteRefreshSegmentTracker.Stats( + shardId, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10, + 10, + 0, + 0, + 0, + 0, + 10, + 10, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 10, + 10, + 10, + 10, + 0 + ); + } + + static RemoteRefreshSegmentTracker.Stats createStatsForRemoteStoreRestoredPrimary(ShardId shardId) { + return new RemoteRefreshSegmentTracker.Stats( + shardId, + 50, + 50, + 0, + 50, + 11, + 11, + 10, + 10, + 0, + 10, + 10, + 0, + 10, + 10, + 0, + 10, + 10, + 0, + 0, + 0, + 100, + 12, + 2, + 2, + 10, + 10, + 10, + 10, + 0 + ); + } + static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); } static void compareStatsResponse( Map statsObject, - RemoteRefreshSegmentTracker.Stats pressureTrackerStats, + RemoteRefreshSegmentTracker.Stats statsTracker, ShardRouting routing ) { assertEquals( @@ -85,156 +155,162 @@ static void compareStatsResponse( Map segmentDownloads = ((Map) segment.get(RemoteStoreStats.SubFields.DOWNLOAD)); Map segmentUploads = ((Map) segment.get(RemoteStoreStats.SubFields.UPLOAD)); - if (!routing.primary()) { + if (statsTracker.totalDownloadsStarted != 0) { assertEquals( - segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_DOWNLOAD_TIMESTAMP), - (int) pressureTrackerStats.lastDownloadTimestampMs + segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_SYNC_TIMESTAMP), + (int) statsTracker.lastDownloadTimestampMs ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( RemoteStoreStats.SubFields.STARTED ), - (int) pressureTrackerStats.totalDownloadsStarted + (int) statsTracker.totalDownloadsStarted ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( RemoteStoreStats.SubFields.SUCCEEDED ), - (int) pressureTrackerStats.totalDownloadsSucceeded + (int) statsTracker.totalDownloadsSucceeded ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( RemoteStoreStats.SubFields.FAILED ), - (int) pressureTrackerStats.totalDownloadsFailed + (int) statsTracker.totalDownloadsFailed ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.STARTED ), - (int) pressureTrackerStats.downloadBytesStarted + (int) statsTracker.downloadBytesStarted ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.SUCCEEDED ), - (int) pressureTrackerStats.downloadBytesSucceeded + (int) statsTracker.downloadBytesSucceeded ); assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_FILE_DOWNLOADS_IN_BYTES)).get( + ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.FAILED ), - (int) pressureTrackerStats.downloadBytesFailed + (int) statsTracker.downloadBytesFailed ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.LAST_SUCCESSFUL ), - (int) pressureTrackerStats.lastSuccessfulSegmentDownloadBytes + (int) statsTracker.lastSuccessfulSegmentDownloadBytes ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - pressureTrackerStats.downloadBytesMovingAverage + statsTracker.downloadBytesMovingAverage ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - pressureTrackerStats.downloadBytesPerSecMovingAverage + statsTracker.downloadBytesPerSecMovingAverage ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - pressureTrackerStats.downloadTimeMovingAverage + statsTracker.downloadTimeMovingAverage ); - assertTrue(segmentUploads.isEmpty()); } else { assertTrue(segmentDownloads.isEmpty()); + } + + if (statsTracker.totalUploadsStarted != 0) { assertEquals( segmentUploads.get(RemoteStoreStats.UploadStatsFields.LOCAL_REFRESH_TIMESTAMP), - (int) pressureTrackerStats.localRefreshClockTimeMs + (int) statsTracker.localRefreshClockTimeMs ); assertEquals( segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_TIMESTAMP), - (int) pressureTrackerStats.remoteRefreshClockTimeMs + (int) statsTracker.remoteRefreshClockTimeMs ); assertEquals( segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS), - (int) pressureTrackerStats.refreshTimeLagMs + (int) statsTracker.refreshTimeLagMs ); assertEquals( segmentUploads.get(RemoteStoreStats.UploadStatsFields.REFRESH_LAG), - (int) (pressureTrackerStats.localRefreshNumber - pressureTrackerStats.remoteRefreshNumber) + (int) (statsTracker.localRefreshNumber - statsTracker.remoteRefreshNumber) ); - assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.BYTES_LAG), (int) pressureTrackerStats.bytesLag); + assertEquals(segmentUploads.get(RemoteStoreStats.UploadStatsFields.BYTES_LAG), (int) statsTracker.bytesLag); assertEquals( segmentUploads.get(RemoteStoreStats.UploadStatsFields.BACKPRESSURE_REJECTION_COUNT), - (int) pressureTrackerStats.rejectionCount + (int) statsTracker.rejectionCount ); assertEquals( segmentUploads.get(RemoteStoreStats.UploadStatsFields.CONSECUTIVE_FAILURE_COUNT), - (int) pressureTrackerStats.consecutiveFailuresCount + (int) statsTracker.consecutiveFailuresCount ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.STARTED ), - (int) pressureTrackerStats.uploadBytesStarted + (int) statsTracker.uploadBytesStarted ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.SUCCEEDED ), - (int) pressureTrackerStats.uploadBytesSucceeded + (int) statsTracker.uploadBytesSucceeded ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_UPLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.FAILED ), - (int) pressureTrackerStats.uploadBytesFailed + (int) statsTracker.uploadBytesFailed ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - pressureTrackerStats.uploadBytesMovingAverage + statsTracker.uploadBytesMovingAverage ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.LAST_SUCCESSFUL ), - (int) pressureTrackerStats.lastSuccessfulRemoteRefreshBytes + (int) statsTracker.lastSuccessfulRemoteRefreshBytes ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - pressureTrackerStats.uploadBytesPerSecMovingAverage + statsTracker.uploadBytesPerSecMovingAverage ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.STARTED), - (int) pressureTrackerStats.totalUploadsStarted + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_SYNCS_TO_REMOTE)).get( + RemoteStoreStats.SubFields.STARTED + ), + (int) statsTracker.totalUploadsStarted ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get( + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_SYNCS_TO_REMOTE)).get( RemoteStoreStats.SubFields.SUCCEEDED ), - (int) pressureTrackerStats.totalUploadsSucceeded + (int) statsTracker.totalUploadsSucceeded ); assertEquals( - ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_REMOTE_REFRESH)).get(RemoteStoreStats.SubFields.FAILED), - (int) pressureTrackerStats.totalUploadsFailed + ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.TOTAL_SYNCS_TO_REMOTE)).get(RemoteStoreStats.SubFields.FAILED), + (int) statsTracker.totalUploadsFailed ); assertEquals( ((Map) segmentUploads.get(RemoteStoreStats.UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - pressureTrackerStats.uploadTimeMovingAverage + statsTracker.uploadTimeMovingAverage ); + } else { + assertTrue(segmentUploads.isEmpty()); } } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 2e421a5b08a33..3ba9733e1800b 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -25,8 +25,10 @@ import java.util.Map; import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.compareStatsResponse; -import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createPressureTrackerStats; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createStatsForNewReplica; import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createShardRouting; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createStatsForNewPrimary; +import static org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsTestHelper.createStatsForRemoteStoreRestoredPrimary; import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; public class RemoteStoreStatsTests extends OpenSearchTestCase { @@ -47,61 +49,139 @@ public void tearDown() throws Exception { } public void testXContentBuilderWithPrimaryShard() throws IOException { - RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); + RemoteRefreshSegmentTracker.Stats uploadStats = createStatsForNewPrimary(shardId); ShardRouting routing = createShardRouting(shardId, true); - RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats, routing); + RemoteStoreStats stats = new RemoteStoreStats(uploadStats, routing); XContentBuilder builder = XContentFactory.jsonBuilder(); stats.toXContent(builder, EMPTY_PARAMS); Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); - compareStatsResponse(jsonObject, pressureTrackerStats, routing); + compareStatsResponse(jsonObject, uploadStats, routing); } public void testXContentBuilderWithReplicaShard() throws IOException { - RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); + RemoteRefreshSegmentTracker.Stats downloadStats = createStatsForNewReplica(shardId); ShardRouting routing = createShardRouting(shardId, false); - RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats, routing); + RemoteStoreStats stats = new RemoteStoreStats(downloadStats, routing); XContentBuilder builder = XContentFactory.jsonBuilder(); stats.toXContent(builder, EMPTY_PARAMS); Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); - compareStatsResponse(jsonObject, pressureTrackerStats, routing); + compareStatsResponse(jsonObject, downloadStats, routing); } - public void testSerialization() throws Exception { - RemoteRefreshSegmentTracker.Stats pressureTrackerStats = createPressureTrackerStats(shardId); - RemoteStoreStats stats = new RemoteStoreStats(pressureTrackerStats, createShardRouting(shardId, true)); + public void testXContentBuilderWithRemoteStoreRestoredShard() throws IOException { + RemoteRefreshSegmentTracker.Stats remotestoreRestoredShardStats = createStatsForRemoteStoreRestoredPrimary(shardId); + ShardRouting routing = createShardRouting(shardId, true); + RemoteStoreStats stats = new RemoteStoreStats(remotestoreRestoredShardStats, routing); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + stats.toXContent(builder, EMPTY_PARAMS); + Map jsonObject = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + compareStatsResponse(jsonObject, remotestoreRestoredShardStats, routing); + } + + public void testSerializationForPrimaryShard() throws Exception { + RemoteRefreshSegmentTracker.Stats primaryShardStats = createStatsForNewPrimary(shardId); + RemoteStoreStats stats = new RemoteStoreStats(primaryShardStats, createShardRouting(shardId, true)); + try (BytesStreamOutput out = new BytesStreamOutput()) { + stats.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); + assertEquals(stats.getStats().refreshTimeLagMs, deserializedStats.refreshTimeLagMs); + assertEquals(stats.getStats().localRefreshNumber, deserializedStats.localRefreshNumber); + assertEquals(stats.getStats().remoteRefreshNumber, deserializedStats.remoteRefreshNumber); + assertEquals(stats.getStats().uploadBytesStarted, deserializedStats.uploadBytesStarted); + assertEquals(stats.getStats().uploadBytesSucceeded, deserializedStats.uploadBytesSucceeded); + assertEquals(stats.getStats().uploadBytesFailed, deserializedStats.uploadBytesFailed); + assertEquals(stats.getStats().totalUploadsStarted, deserializedStats.totalUploadsStarted); + assertEquals(stats.getStats().totalUploadsFailed, deserializedStats.totalUploadsFailed); + assertEquals(stats.getStats().totalUploadsSucceeded, deserializedStats.totalUploadsSucceeded); + assertEquals(stats.getStats().rejectionCount, deserializedStats.rejectionCount); + assertEquals(stats.getStats().consecutiveFailuresCount, deserializedStats.consecutiveFailuresCount); + assertEquals(stats.getStats().uploadBytesMovingAverage, deserializedStats.uploadBytesMovingAverage, 0); + assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); + assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); + assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); + assertEquals(0, deserializedStats.totalDownloadsStarted); + assertEquals(0, deserializedStats.totalDownloadsSucceeded); + assertEquals(0, deserializedStats.totalDownloadsFailed); + assertEquals(0, deserializedStats.downloadBytesStarted); + assertEquals(0, deserializedStats.downloadBytesFailed); + assertEquals(0, deserializedStats.downloadBytesSucceeded); + assertEquals(0, deserializedStats.lastSuccessfulSegmentDownloadBytes); + assertEquals(0, deserializedStats.lastDownloadTimestampMs); + } + } + } + + public void testSerializationForReplicaShard() throws Exception { + RemoteRefreshSegmentTracker.Stats replicaShardStats = createStatsForNewReplica(shardId); + RemoteStoreStats stats = new RemoteStoreStats(replicaShardStats, createShardRouting(shardId, false)); + try (BytesStreamOutput out = new BytesStreamOutput()) { + stats.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); + assertEquals(0, deserializedStats.refreshTimeLagMs); + assertEquals(0, deserializedStats.localRefreshNumber); + assertEquals(0, deserializedStats.remoteRefreshNumber); + assertEquals(0, deserializedStats.uploadBytesStarted); + assertEquals(0, deserializedStats.uploadBytesSucceeded); + assertEquals(0, deserializedStats.uploadBytesFailed); + assertEquals(0, deserializedStats.totalUploadsStarted); + assertEquals(0, deserializedStats.totalUploadsFailed); + assertEquals(0, deserializedStats.totalUploadsSucceeded); + assertEquals(0, deserializedStats.rejectionCount); + assertEquals(0, deserializedStats.consecutiveFailuresCount); + assertEquals(0, deserializedStats.bytesLag); + assertEquals(stats.getStats().totalDownloadsStarted, deserializedStats.totalDownloadsStarted); + assertEquals(stats.getStats().totalDownloadsSucceeded, deserializedStats.totalDownloadsSucceeded); + assertEquals(stats.getStats().totalDownloadsFailed, deserializedStats.totalDownloadsFailed); + assertEquals(stats.getStats().downloadBytesStarted, deserializedStats.downloadBytesStarted); + assertEquals(stats.getStats().downloadBytesFailed, deserializedStats.downloadBytesFailed); + assertEquals(stats.getStats().downloadBytesSucceeded, deserializedStats.downloadBytesSucceeded); + assertEquals(stats.getStats().lastSuccessfulSegmentDownloadBytes, deserializedStats.lastSuccessfulSegmentDownloadBytes); + assertEquals(stats.getStats().lastDownloadTimestampMs, deserializedStats.lastDownloadTimestampMs); + assertEquals(stats.getStats().downloadBytesPerSecMovingAverage, deserializedStats.downloadBytesPerSecMovingAverage, 0); + assertEquals(stats.getStats().downloadTimeMovingAverage, deserializedStats.downloadTimeMovingAverage, 0); + assertEquals(stats.getStats().downloadBytesMovingAverage, deserializedStats.downloadBytesMovingAverage, 0); + } + } + } + + public void testSerializationForRemoteStoreRestoredPrimaryShard() throws Exception { + RemoteRefreshSegmentTracker.Stats primaryShardStats = createStatsForRemoteStoreRestoredPrimary(shardId); + RemoteStoreStats stats = new RemoteStoreStats(primaryShardStats, createShardRouting(shardId, true)); try (BytesStreamOutput out = new BytesStreamOutput()) { stats.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); - assertEquals(deserializedStats.shardId.toString(), stats.getStats().shardId.toString()); - assertEquals(deserializedStats.refreshTimeLagMs, stats.getStats().refreshTimeLagMs); - assertEquals(deserializedStats.localRefreshNumber, stats.getStats().localRefreshNumber); - assertEquals(deserializedStats.remoteRefreshNumber, stats.getStats().remoteRefreshNumber); - assertEquals(deserializedStats.uploadBytesStarted, stats.getStats().uploadBytesStarted); - assertEquals(deserializedStats.uploadBytesSucceeded, stats.getStats().uploadBytesSucceeded); - assertEquals(deserializedStats.uploadBytesFailed, stats.getStats().uploadBytesFailed); - assertEquals(deserializedStats.totalUploadsStarted, stats.getStats().totalUploadsStarted); - assertEquals(deserializedStats.totalUploadsFailed, stats.getStats().totalUploadsFailed); - assertEquals(deserializedStats.totalUploadsSucceeded, stats.getStats().totalUploadsSucceeded); - assertEquals(deserializedStats.rejectionCount, stats.getStats().rejectionCount); - assertEquals(deserializedStats.consecutiveFailuresCount, stats.getStats().consecutiveFailuresCount); - assertEquals(deserializedStats.uploadBytesMovingAverage, stats.getStats().uploadBytesMovingAverage, 0); - assertEquals(deserializedStats.uploadBytesPerSecMovingAverage, stats.getStats().uploadBytesPerSecMovingAverage, 0); - assertEquals(deserializedStats.uploadTimeMovingAverage, stats.getStats().uploadTimeMovingAverage, 0); - assertEquals(deserializedStats.bytesLag, stats.getStats().bytesLag); - assertEquals(deserializedStats.totalDownloadsStarted, stats.getStats().totalDownloadsStarted); - assertEquals(deserializedStats.totalDownloadsSucceeded, stats.getStats().totalDownloadsSucceeded); - assertEquals(deserializedStats.totalDownloadsFailed, stats.getStats().totalDownloadsFailed); - assertEquals(deserializedStats.downloadBytesStarted, stats.getStats().downloadBytesStarted); - assertEquals(deserializedStats.downloadBytesFailed, stats.getStats().downloadBytesFailed); - assertEquals(deserializedStats.downloadBytesSucceeded, stats.getStats().downloadBytesSucceeded); - assertEquals(deserializedStats.lastSuccessfulSegmentDownloadBytes, stats.getStats().lastSuccessfulSegmentDownloadBytes); - assertEquals(deserializedStats.lastDownloadTimestampMs, stats.getStats().lastDownloadTimestampMs); - assertEquals(deserializedStats.downloadBytesMovingAverage, stats.getStats().downloadBytesMovingAverage, 0); - assertEquals(deserializedStats.downloadBytesPerSecMovingAverage, stats.getStats().downloadBytesPerSecMovingAverage, 0); - assertEquals(deserializedStats.downloadTimeMovingAverage, stats.getStats().downloadTimeMovingAverage, 0); + assertEquals(stats.getStats().refreshTimeLagMs, deserializedStats.refreshTimeLagMs); + assertEquals(stats.getStats().localRefreshNumber, deserializedStats.localRefreshNumber); + assertEquals(stats.getStats().remoteRefreshNumber, deserializedStats.remoteRefreshNumber); + assertEquals(stats.getStats().uploadBytesStarted, deserializedStats.uploadBytesStarted); + assertEquals(stats.getStats().uploadBytesSucceeded, deserializedStats.uploadBytesSucceeded); + assertEquals(stats.getStats().uploadBytesFailed, deserializedStats.uploadBytesFailed); + assertEquals(stats.getStats().totalUploadsStarted, deserializedStats.totalUploadsStarted); + assertEquals(stats.getStats().totalUploadsFailed, deserializedStats.totalUploadsFailed); + assertEquals(stats.getStats().totalUploadsSucceeded, deserializedStats.totalUploadsSucceeded); + assertEquals(stats.getStats().rejectionCount, deserializedStats.rejectionCount); + assertEquals(stats.getStats().consecutiveFailuresCount, deserializedStats.consecutiveFailuresCount); + assertEquals(stats.getStats().uploadBytesMovingAverage, deserializedStats.uploadBytesMovingAverage, 0); + assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); + assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); + assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); + assertEquals(stats.getStats().totalDownloadsStarted, deserializedStats.totalDownloadsStarted); + assertEquals(stats.getStats().totalDownloadsSucceeded, deserializedStats.totalDownloadsSucceeded); + assertEquals(stats.getStats().totalDownloadsFailed, deserializedStats.totalDownloadsFailed); + assertEquals(stats.getStats().downloadBytesStarted, deserializedStats.downloadBytesStarted); + assertEquals(stats.getStats().downloadBytesFailed, deserializedStats.downloadBytesFailed); + assertEquals(stats.getStats().downloadBytesSucceeded, deserializedStats.downloadBytesSucceeded); + assertEquals(stats.getStats().lastSuccessfulSegmentDownloadBytes, deserializedStats.lastSuccessfulSegmentDownloadBytes); + assertEquals(stats.getStats().lastDownloadTimestampMs, deserializedStats.lastDownloadTimestampMs); + assertEquals(stats.getStats().downloadBytesPerSecMovingAverage, deserializedStats.downloadBytesPerSecMovingAverage, 0); + assertEquals(stats.getStats().downloadTimeMovingAverage, deserializedStats.downloadTimeMovingAverage, 0); + assertEquals(stats.getStats().downloadBytesMovingAverage, deserializedStats.downloadBytesMovingAverage, 0); } } } From 0d00a798fbaf3a012713a894fbb4d5e512b33603 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:26:43 +0530 Subject: [PATCH 17/36] Adding more integ tests on stats correctness Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index d0578faf1b563..cb56969444453 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -474,6 +474,33 @@ public void testStatsOnRemoteStoreRestore() throws IOException { }); } + public void testNonZeroStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception { + // Create an index with one primary and one replica shard + createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); + ensureGreen(INDEX_NAME); + + // Ensure that the index has 0 documents in it + assertEquals(0, client().admin().indices().prepareStats(INDEX_NAME).get().getTotal().docs.getCount()); + + // Assert that within 5 seconds the download and upload stats moves to a non-zero value + assertBusy(() -> { + RemoteStoreStats[] remoteStoreStats = client().admin() + .cluster() + .prepareRemoteStoreStats(INDEX_NAME, "0") + .get() + .getRemoteStoreStats(); + Arrays.stream(remoteStoreStats).forEach(statObject -> { + RemoteRefreshSegmentTracker.Stats segmentTracker = statObject.getStats(); + if (statObject.getShardRouting().primary()) { + assertTrue(segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0); + } else { + assertTrue(segmentTracker.totalDownloadsStarted > 0 && segmentTracker.totalDownloadsSucceeded > 0 + && segmentTracker.totalDownloadsFailed == 0); + } + }); + }, 5, TimeUnit.SECONDS); + } + private void indexDocs(boolean withFlushAndRefresh) { if (withFlushAndRefresh) { // Indexing documents along with refreshes and flushes. @@ -495,13 +522,6 @@ private void indexDocs(boolean withFlushAndRefresh) { } } - private IndexResponse indexSingleDoc() { - return client().prepareIndex(INDEX_NAME) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - private void changeReplicaCountAndEnsureGreen(int replicaCount) { assertAcked( client().admin() From 41497f33de04211cc2286e5ebb8eecb5ddd35a20 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:47:22 +0530 Subject: [PATCH 18/36] Fixed Spotless checks Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index cb56969444453..329b7e24e469d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -13,14 +13,12 @@ import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStats; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsRequestBuilder; import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsResponse; -import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; -import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; @@ -277,7 +275,7 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce assertEquals(uploadBytesAverage, downloadBytesAverage, 0); // Assert last segment size uploaded = last segment size downloaded assertEquals(lastDownloadedSegmentSize, lastUploadedSegmentSize); - }, 30, TimeUnit.SECONDS); + }, 1, TimeUnit.MINUTES); } } @@ -354,7 +352,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr assertEquals(uploadBytesAverage, downloadBytesAverage.get(j), 0); assertEquals(lastUploadedSegmentSize, (long) lastDownloadedSegmentSize.get(j)); } - }, 45, TimeUnit.SECONDS); + }, 1, TimeUnit.MINUTES); } } @@ -492,10 +490,17 @@ public void testNonZeroStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception { Arrays.stream(remoteStoreStats).forEach(statObject -> { RemoteRefreshSegmentTracker.Stats segmentTracker = statObject.getStats(); if (statObject.getShardRouting().primary()) { - assertTrue(segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0); + assertTrue( + segmentTracker.totalUploadsStarted > 0 + && segmentTracker.totalUploadsSucceeded > 0 + && segmentTracker.totalUploadsFailed == 0 + ); } else { - assertTrue(segmentTracker.totalDownloadsStarted > 0 && segmentTracker.totalDownloadsSucceeded > 0 - && segmentTracker.totalDownloadsFailed == 0); + assertTrue( + segmentTracker.totalDownloadsStarted > 0 + && segmentTracker.totalDownloadsSucceeded > 0 + && segmentTracker.totalDownloadsFailed == 0 + ); } }); }, 5, TimeUnit.SECONDS); From 7a4d99349564e789b8f7be55df54379c4414727b Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Wed, 19 Jul 2023 11:48:41 +0530 Subject: [PATCH 19/36] Addressing comments Signed-off-by: Ashish Singh --- .../remotestore/RemoteStoreStatsIT.java | 63 +++++++------------ 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 329b7e24e469d..6c8dc8e3f613b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -231,50 +231,35 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce // Poll for RemoteStore Stats assertBusy(() -> { RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; - long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; - long downloadsStarted = 0, downloadsSucceeded = 0, downloadsFailed = 0; - long downloadBytesStarted = 0, downloadBytesSucceeded = 0, downloadBytesFailed = 0; - double uploadBytesAverage = 0, downloadBytesAverage = 0; - long lastUploadedSegmentSize = 0, lastDownloadedSegmentSize = 0; - // Iterate through the response and extract the relevant segment upload and download stats - for (RemoteStoreStats eachStatsObject : response.getRemoteStoreStats()) { - RemoteRefreshSegmentTracker.Stats stats = eachStatsObject.getStats(); - if (eachStatsObject.getShardRouting().primary()) { - uploadsStarted = stats.totalUploadsStarted; - uploadsSucceeded = stats.totalUploadsSucceeded; - uploadsFailed = stats.totalUploadsFailed; - uploadBytesStarted = stats.uploadBytesStarted; - uploadBytesSucceeded = stats.uploadBytesSucceeded; - uploadBytesFailed = stats.uploadBytesFailed; - uploadBytesAverage = stats.uploadBytesMovingAverage; - lastUploadedSegmentSize = stats.lastSuccessfulRemoteRefreshBytes; - } else { - downloadsStarted = stats.totalDownloadsStarted; - downloadsSucceeded = stats.totalDownloadsSucceeded; - downloadsFailed = stats.totalDownloadsFailed; - downloadBytesStarted = stats.downloadBytesStarted; - downloadBytesSucceeded = stats.downloadBytesSucceeded; - downloadBytesFailed = stats.downloadBytesFailed; - downloadBytesAverage = stats.downloadBytesMovingAverage; - lastDownloadedSegmentSize = stats.lastSuccessfulSegmentDownloadBytes; - } - } + List primaryStatsList = Arrays.stream(response.getRemoteStoreStats()) + .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) + .collect(Collectors.toList()); + assertEquals(1, primaryStatsList.size()); + List replicaStatsList = Arrays.stream(response.getRemoteStoreStats()) + .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) + .collect(Collectors.toList()); + assertEquals(1, replicaStatsList.size()); + RemoteRefreshSegmentTracker.Stats primaryStats = primaryStatsList.get(0).getStats(); + RemoteRefreshSegmentTracker.Stats replicaStats = replicaStatsList.get(0).getStats(); // Assert Upload syncs = download syncs - assertTrue(uploadsStarted > 0 && uploadsStarted == downloadsStarted); - assertTrue(uploadsSucceeded > 0 && uploadsSucceeded == downloadsSucceeded); - assertTrue(downloadBytesStarted > 0 && uploadBytesStarted == downloadBytesStarted); - assertTrue(downloadBytesSucceeded > 0 && uploadBytesSucceeded == downloadBytesSucceeded); + assertTrue(primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted == replicaStats.totalUploadsStarted); + assertTrue( + primaryStats.totalUploadsSucceeded > 0 && primaryStats.totalUploadsSucceeded == replicaStats.totalDownloadsSucceeded + ); + assertTrue(replicaStats.downloadBytesStarted > 0 && primaryStats.uploadBytesStarted == replicaStats.downloadBytesStarted); + assertTrue( + replicaStats.downloadBytesSucceeded > 0 && primaryStats.uploadBytesSucceeded == replicaStats.downloadBytesSucceeded + ); // Assert zero failures - assertEquals(0, uploadsFailed); - assertEquals(0, uploadBytesFailed); - assertEquals(0, downloadsFailed); - assertEquals(0, downloadBytesFailed); + assertEquals(0, primaryStats.totalDownloadsFailed); + assertEquals(0, primaryStats.uploadBytesFailed); + assertEquals(0, replicaStats.totalDownloadsFailed); + assertEquals(0, replicaStats.downloadBytesFailed); // Assert transfer bytes average is same across downloads and uploads - assertEquals(uploadBytesAverage, downloadBytesAverage, 0); + assertEquals(primaryStats.uploadBytesMovingAverage, replicaStats.downloadBytesMovingAverage, 0); // Assert last segment size uploaded = last segment size downloaded - assertEquals(lastDownloadedSegmentSize, lastUploadedSegmentSize); + assertEquals(replicaStats.lastSuccessfulSegmentDownloadBytes, primaryStats.lastSuccessfulRemoteRefreshBytes); }, 1, TimeUnit.MINUTES); } } From ac7df33d773f326de79eb56100ebf4030dbf0700 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 19 Jul 2023 12:30:05 +0530 Subject: [PATCH 20/36] Retrigger Integs Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> From 70a60f23a1d3433d0e854d4decaffc315c52a1db Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:30:53 +0530 Subject: [PATCH 21/36] Retrigger Integs 2 Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> From df43b9c7b2fd9a650a7272e6d3db6891b2d8ffe5 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 19 Jul 2023 14:18:00 +0530 Subject: [PATCH 22/36] Fixing assertion on Stats UT Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../java/org/opensearch/remotestore/RemoteStoreStatsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 6c8dc8e3f613b..ec6b195a490c8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -243,7 +243,7 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce RemoteRefreshSegmentTracker.Stats primaryStats = primaryStatsList.get(0).getStats(); RemoteRefreshSegmentTracker.Stats replicaStats = replicaStatsList.get(0).getStats(); // Assert Upload syncs = download syncs - assertTrue(primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted == replicaStats.totalUploadsStarted); + assertTrue(primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted == replicaStats.totalDownloadsStarted); assertTrue( primaryStats.totalUploadsSucceeded > 0 && primaryStats.totalUploadsSucceeded == replicaStats.totalDownloadsSucceeded ); From 6e3d1da684e57a59996696400a7fc8631efcd6d8 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:17:43 +0530 Subject: [PATCH 23/36] Fixing stats correctness test cases for RemoteStoreStatsIT Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 224 ++++++++++++------ 1 file changed, 154 insertions(+), 70 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index ec6b195a490c8..9c4a82eaf2313 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -64,7 +64,7 @@ public void testStatsResponseFromAllNodes() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexDocs(true); + indexDocs(); // Step 2 - We find all the nodes that are present in the cluster. We make the remote store stats api call from // each of the node in the cluster and check that the response is coming as expected. @@ -121,7 +121,7 @@ public void testStatsResponseAllShards() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexDocs(true); + indexDocs(); // Step 2 - We find all the nodes that are present in the cluster. We make the remote store stats api call from // each of the node in the cluster and check that the response is coming as expected. @@ -166,7 +166,7 @@ public void testStatsResponseFromLocalNode() { ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - indexDocs(true); + indexDocs(); // Step 2 - We find a data node in the cluster. We make the remote store stats api call from // each of the data node in the cluster and check that only local shards are returned. @@ -215,18 +215,60 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce // - Index documents // - Trigger refresh and flush // - Assert that download stats == upload stats - // - Repeat this step for random times (between 2 and 5) + // - Repeat this step for random times (between 5 and 10) - // Create index with 1 pri and 1 replica - createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); - // Disabling refresh interval - disableRefreshInterval(); + // Create index with 1 pri and 1 replica and refresh interval disabled + createIndex( + INDEX_NAME, + Settings.builder().put(remoteStoreIndexSettings(1, 1)).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1).build() + ); + + // Ensure zero stats when there are no refreshes + RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + for (RemoteStoreStats stats : zeroStateResponse.getRemoteStoreStats()) { + assertZeroStats(stats); + } + + // Manually invoke a refresh + refresh(INDEX_NAME); + + // Get zero state values + // Extract and assert zero state primary stats + zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + RemoteRefreshSegmentTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) + .collect(Collectors.toList()) + .get(0) + .getStats(); + assertTrue( + zeroStatePrimaryStats.totalUploadsStarted == zeroStatePrimaryStats.totalUploadsSucceeded + && zeroStatePrimaryStats.totalUploadsSucceeded == 1 + ); + assertTrue( + zeroStatePrimaryStats.uploadBytesStarted == zeroStatePrimaryStats.uploadBytesSucceeded + && zeroStatePrimaryStats.uploadBytesSucceeded > 0 + ); + assertTrue(zeroStatePrimaryStats.totalUploadsFailed == 0 && zeroStatePrimaryStats.uploadBytesFailed == 0); + + // Extract and assert zero state replica stats + RemoteRefreshSegmentTracker.Stats zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) + .collect(Collectors.toList()) + .get(0) + .getStats(); + assertTrue( + zeroStateReplicaStats.totalDownloadsStarted == 0 + && zeroStateReplicaStats.totalDownloadsSucceeded == 0 + && zeroStateReplicaStats.totalDownloadsFailed == 0 + ); + assertTrue(zeroStateReplicaStats.downloadBytesStarted == 0 && zeroStateReplicaStats.downloadBytesSucceeded == 0); // Index documents - for (int i = 1; i <= randomIntBetween(2, 5); i++) { - indexDocs(false); + for (int i = 1; i <= randomIntBetween(5, 10); i++) { + indexSingleDoc(INDEX_NAME); // Running Flush & Refresh manually flushAndRefresh(INDEX_NAME); + ensureGreen(INDEX_NAME); // Poll for RemoteStore Stats assertBusy(() -> { @@ -242,25 +284,31 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce assertEquals(1, replicaStatsList.size()); RemoteRefreshSegmentTracker.Stats primaryStats = primaryStatsList.get(0).getStats(); RemoteRefreshSegmentTracker.Stats replicaStats = replicaStatsList.get(0).getStats(); - // Assert Upload syncs = download syncs - assertTrue(primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted == replicaStats.totalDownloadsStarted); + // Assert Upload syncs - zero state uploads == download syncs + assertTrue( + primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted - 1 == replicaStats.totalDownloadsStarted + ); assertTrue( - primaryStats.totalUploadsSucceeded > 0 && primaryStats.totalUploadsSucceeded == replicaStats.totalDownloadsSucceeded + primaryStats.totalUploadsSucceeded > 0 && primaryStats.totalUploadsSucceeded - 1 == replicaStats.totalDownloadsSucceeded ); - assertTrue(replicaStats.downloadBytesStarted > 0 && primaryStats.uploadBytesStarted == replicaStats.downloadBytesStarted); assertTrue( - replicaStats.downloadBytesSucceeded > 0 && primaryStats.uploadBytesSucceeded == replicaStats.downloadBytesSucceeded + replicaStats.downloadBytesStarted > 0 + && primaryStats.uploadBytesStarted - zeroStatePrimaryStats.uploadBytesStarted == replicaStats.downloadBytesStarted + ); + assertTrue( + replicaStats.downloadBytesSucceeded > 0 + && primaryStats.uploadBytesSucceeded + - zeroStatePrimaryStats.uploadBytesSucceeded == replicaStats.downloadBytesSucceeded ); // Assert zero failures assertEquals(0, primaryStats.totalDownloadsFailed); assertEquals(0, primaryStats.uploadBytesFailed); assertEquals(0, replicaStats.totalDownloadsFailed); assertEquals(0, replicaStats.downloadBytesFailed); - // Assert transfer bytes average is same across downloads and uploads - assertEquals(primaryStats.uploadBytesMovingAverage, replicaStats.downloadBytesMovingAverage, 0); + // Assert last segment size uploaded = last segment size downloaded assertEquals(replicaStats.lastSuccessfulSegmentDownloadBytes, primaryStats.lastSuccessfulRemoteRefreshBytes); - }, 1, TimeUnit.MINUTES); + }, 30, TimeUnit.SECONDS); } } @@ -271,33 +319,73 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr // - Index documents // - Trigger refresh and flush // - Assert that download stats == upload stats - // - Repeat this step for random times (between 2 and 5) + // - Repeat this step for random times (between 5 and 10) - // Crete index + // Create index int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); - createIndex(INDEX_NAME, remoteStoreIndexSettings(dataNodeCount - 1, 1)); + createIndex( + INDEX_NAME, + Settings.builder() + .put(remoteStoreIndexSettings(dataNodeCount - 1, 1)) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) + .build() + ); - // Disable refresh interval - disableRefreshInterval(); - ensureGreen(INDEX_NAME); - int currentNodesInCluster = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); - for (int i = 0; i < randomIntBetween(2, 5); i++) { - indexDocs(false); + RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + for (RemoteStoreStats stats : zeroStateResponse.getRemoteStoreStats()) { + assertZeroStats(stats); + } + + // Manually invoke a refresh + refresh(INDEX_NAME); + + // Get zero state values + // Extract and assert zero state primary stats + zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + RemoteRefreshSegmentTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) + .collect(Collectors.toList()) + .get(0) + .getStats(); + assertTrue( + zeroStatePrimaryStats.totalUploadsStarted == zeroStatePrimaryStats.totalUploadsSucceeded + && zeroStatePrimaryStats.totalUploadsSucceeded == 1 + ); + assertTrue( + zeroStatePrimaryStats.uploadBytesStarted == zeroStatePrimaryStats.uploadBytesSucceeded + && zeroStatePrimaryStats.uploadBytesSucceeded > 0 + ); + assertTrue(zeroStatePrimaryStats.totalUploadsFailed == 0 && zeroStatePrimaryStats.uploadBytesFailed == 0); + + // Extract and assert zero state replica stats + List zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) + .collect(Collectors.toList()); + zeroStateReplicaStats.forEach(stats -> { + assertTrue( + stats.getStats().totalDownloadsStarted == 0 + && stats.getStats().totalDownloadsSucceeded == 0 + && stats.getStats().totalDownloadsFailed == 0 + ); + assertTrue(stats.getStats().downloadBytesStarted == 0 && stats.getStats().downloadBytesSucceeded == 0); + }); + int currentNodesInCluster = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); + for (int i = 0; i < randomIntBetween(5, 10); i++) { + indexSingleDoc(INDEX_NAME); // Running Flush & Refresh manually flushAndRefresh(INDEX_NAME); + ensureGreen(INDEX_NAME); assertBusy(() -> { RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); assertEquals(currentNodesInCluster, response.getSuccessfulShards()); long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; - double uploadBytesAverage = 0; List downloadsStarted = new ArrayList<>(), downloadsSucceeded = new ArrayList<>(), downloadsFailed = new ArrayList<>(); List downloadBytesStarted = new ArrayList<>(), downloadBytesSucceeded = new ArrayList<>(), downloadBytesFailed = new ArrayList<>(); - List downloadBytesAverage = new ArrayList<>(); long lastUploadedSegmentSize = 0; List lastDownloadedSegmentSize = new ArrayList<>(); @@ -311,7 +399,6 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr uploadBytesStarted = stats.uploadBytesStarted; uploadBytesSucceeded = stats.uploadBytesSucceeded; uploadBytesFailed = stats.uploadBytesFailed; - uploadBytesAverage = stats.uploadBytesMovingAverage; lastUploadedSegmentSize = stats.lastSuccessfulRemoteRefreshBytes; } else { downloadsStarted.add(stats.totalDownloadsStarted); @@ -320,7 +407,6 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr downloadBytesStarted.add(stats.downloadBytesStarted); downloadBytesSucceeded.add(stats.downloadBytesSucceeded); downloadBytesFailed.add(stats.downloadBytesFailed); - downloadBytesAverage.add(stats.downloadBytesMovingAverage); lastDownloadedSegmentSize.add(stats.lastSuccessfulSegmentDownloadBytes); } } @@ -328,16 +414,15 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr assertEquals(0, uploadsFailed); assertEquals(0, uploadBytesFailed); for (int j = 0; j < response.getSuccessfulShards() - 1; j++) { - assertEquals(uploadsStarted, (long) downloadsStarted.get(j)); - assertEquals(uploadsSucceeded, (long) downloadsSucceeded.get(j)); + assertEquals(uploadsStarted - 1, (long) downloadsStarted.get(j)); + assertEquals(uploadsSucceeded - 1, (long) downloadsSucceeded.get(j)); assertEquals(0, (long) downloadsFailed.get(j)); - assertEquals(uploadBytesStarted, (long) downloadBytesStarted.get(j)); - assertEquals(uploadBytesSucceeded, (long) downloadBytesSucceeded.get(j)); + assertEquals(uploadBytesStarted - zeroStatePrimaryStats.uploadBytesStarted, (long) downloadBytesStarted.get(j)); + assertEquals(uploadBytesSucceeded - zeroStatePrimaryStats.uploadBytesSucceeded, (long) downloadBytesSucceeded.get(j)); assertEquals(0, (long) downloadBytesFailed.get(j)); - assertEquals(uploadBytesAverage, downloadBytesAverage.get(j), 0); assertEquals(lastUploadedSegmentSize, (long) lastDownloadedSegmentSize.get(j)); } - }, 1, TimeUnit.MINUTES); + }, 30, TimeUnit.SECONDS); } } @@ -351,7 +436,7 @@ public void testStatsOnShardRelocation() { // Create index createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); // Index docs - indexDocs(true); + indexDocs(); // Fetch current set of nodes in the cluster List currentNodesInCluster = getClusterState().nodes() @@ -404,7 +489,7 @@ public void testStatsOnShardUnassigned() throws IOException { // a. Total shard Count in the response object is equal to the previous node count // b. Successful shard count in the response object is equal to the new node count createIndex(INDEX_NAME, remoteStoreIndexSettings(2, 1)); - indexDocs(true); + indexDocs(); int dataNodeCountBeforeStop = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); internalCluster().stopRandomDataNode(); RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); @@ -419,7 +504,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { createIndex(INDEX_NAME, remoteStoreIndexSettings(0, dataNodeCount)); // Index some docs to ensure segments being uploaded to remote store - indexDocs(true); + indexDocs(); // Stop one data node to force the index into a red state internalCluster().stopRandomDataNode(); @@ -436,7 +521,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { ensureGreen(INDEX_NAME); // Index some more docs to force segment uploads to remote store - indexDocs(true); + indexDocs(); RemoteStoreStats[] remoteStoreStats = client().admin() .cluster() @@ -457,7 +542,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { }); } - public void testNonZeroStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception { + public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception { // Create an index with one primary and one replica shard createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); ensureGreen(INDEX_NAME); @@ -476,14 +561,14 @@ public void testNonZeroStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception { RemoteRefreshSegmentTracker.Stats segmentTracker = statObject.getStats(); if (statObject.getShardRouting().primary()) { assertTrue( - segmentTracker.totalUploadsStarted > 0 - && segmentTracker.totalUploadsSucceeded > 0 + segmentTracker.totalUploadsSucceeded == 1 + && segmentTracker.totalUploadsStarted == segmentTracker.totalUploadsSucceeded && segmentTracker.totalUploadsFailed == 0 ); } else { assertTrue( - segmentTracker.totalDownloadsStarted > 0 - && segmentTracker.totalDownloadsSucceeded > 0 + segmentTracker.totalDownloadsStarted == 0 + && segmentTracker.totalDownloadsSucceeded == 0 && segmentTracker.totalDownloadsFailed == 0 ); } @@ -491,22 +576,15 @@ public void testNonZeroStatsOnNewlyCreatedIndexWithZeroDocs() throws Exception { }, 5, TimeUnit.SECONDS); } - private void indexDocs(boolean withFlushAndRefresh) { - if (withFlushAndRefresh) { - // Indexing documents along with refreshes and flushes. - for (int i = 0; i < randomIntBetween(5, 10); i++) { - if (randomBoolean()) { - flush(INDEX_NAME); - } else { - refresh(INDEX_NAME); - } - int numberOfOperations = randomIntBetween(20, 50); - for (int j = 0; j < numberOfOperations; j++) { - indexSingleDoc(INDEX_NAME); - } + private void indexDocs() { + for (int i = 0; i < randomIntBetween(5, 10); i++) { + if (randomBoolean()) { + flush(INDEX_NAME); + } else { + refresh(INDEX_NAME); } - } else { - for (int j = 0; j < randomIntBetween(5, 10); j++) { + int numberOfOperations = randomIntBetween(20, 50); + for (int j = 0; j < numberOfOperations; j++) { indexSingleDoc(INDEX_NAME); } } @@ -528,15 +606,6 @@ private void relocateShard(int shardId, String sourceNode, String destNode) { ensureGreen(INDEX_NAME); } - private void disableRefreshInterval() { - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings(INDEX_NAME) - .setSettings(Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)) - ); - } - private void validateUploadStats(RemoteRefreshSegmentTracker.Stats stats) { assertEquals(0, stats.refreshTimeLagMs); assertEquals(stats.localRefreshNumber, stats.remoteRefreshNumber); @@ -585,4 +654,19 @@ private void validateShardRouting(ShardRouting routing) { ) ); } + + private static void assertZeroStats(RemoteStoreStats stats) { + RemoteRefreshSegmentTracker.Stats statsTracker = stats.getStats(); + if (stats.getShardRouting().primary()) { + assertEquals(0, statsTracker.totalUploadsStarted); + assertEquals(0, statsTracker.totalUploadsSucceeded); + assertEquals(0, statsTracker.uploadBytesStarted); + assertEquals(0, statsTracker.uploadBytesFailed); + } else { + assertEquals(0, statsTracker.totalDownloadsStarted); + assertEquals(0, statsTracker.totalDownloadsSucceeded); + assertEquals(0, statsTracker.downloadBytesStarted); + assertEquals(0, statsTracker.downloadBytesSucceeded); + } + } } From bf43cb14efa1be6c29d6db8164e367e4867a4e0b Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Wed, 19 Jul 2023 22:59:24 +0530 Subject: [PATCH 24/36] Fixing stats IT flakyness Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 45 +++++-------------- 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 9c4a82eaf2313..6cbe0ea97b1d3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -222,19 +222,11 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce INDEX_NAME, Settings.builder().put(remoteStoreIndexSettings(1, 1)).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1).build() ); - - // Ensure zero stats when there are no refreshes - RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - for (RemoteStoreStats stats : zeroStateResponse.getRemoteStoreStats()) { - assertZeroStats(stats); - } - - // Manually invoke a refresh - refresh(INDEX_NAME); + ensureGreen(INDEX_NAME); // Get zero state values // Extract and assert zero state primary stats - zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); RemoteRefreshSegmentTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()) @@ -308,7 +300,7 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce // Assert last segment size uploaded = last segment size downloaded assertEquals(replicaStats.lastSuccessfulSegmentDownloadBytes, primaryStats.lastSuccessfulRemoteRefreshBytes); - }, 30, TimeUnit.SECONDS); + }, 60, TimeUnit.SECONDS); } } @@ -326,22 +318,18 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr createIndex( INDEX_NAME, Settings.builder() - .put(remoteStoreIndexSettings(dataNodeCount - 1, 1)) + .put(remoteStoreIndexSettings(dataNodeCount, 1)) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) .build() ); - - RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - for (RemoteStoreStats stats : zeroStateResponse.getRemoteStoreStats()) { - assertZeroStats(stats); - } + ensureGreen(INDEX_NAME); // Manually invoke a refresh refresh(INDEX_NAME); // Get zero state values // Extract and assert zero state primary stats - zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); RemoteRefreshSegmentTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()) @@ -375,7 +363,6 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr indexSingleDoc(INDEX_NAME); // Running Flush & Refresh manually flushAndRefresh(INDEX_NAME); - ensureGreen(INDEX_NAME); assertBusy(() -> { RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); @@ -422,7 +409,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr assertEquals(0, (long) downloadBytesFailed.get(j)); assertEquals(lastUploadedSegmentSize, (long) lastDownloadedSegmentSize.get(j)); } - }, 30, TimeUnit.SECONDS); + }, 60, TimeUnit.SECONDS); } } @@ -435,6 +422,7 @@ public void testStatsOnShardRelocation() { // Create index createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); + ensureGreen(INDEX_NAME); // Index docs indexDocs(); @@ -489,6 +477,7 @@ public void testStatsOnShardUnassigned() throws IOException { // a. Total shard Count in the response object is equal to the previous node count // b. Successful shard count in the response object is equal to the new node count createIndex(INDEX_NAME, remoteStoreIndexSettings(2, 1)); + ensureGreen(INDEX_NAME); indexDocs(); int dataNodeCountBeforeStop = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); internalCluster().stopRandomDataNode(); @@ -502,6 +491,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { // Creating an index with primary shard count == total nodes in cluster and 0 replicas int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0, dataNodeCount)); + ensureGreen(INDEX_NAME); // Index some docs to ensure segments being uploaded to remote store indexDocs(); @@ -654,19 +644,4 @@ private void validateShardRouting(ShardRouting routing) { ) ); } - - private static void assertZeroStats(RemoteStoreStats stats) { - RemoteRefreshSegmentTracker.Stats statsTracker = stats.getStats(); - if (stats.getShardRouting().primary()) { - assertEquals(0, statsTracker.totalUploadsStarted); - assertEquals(0, statsTracker.totalUploadsSucceeded); - assertEquals(0, statsTracker.uploadBytesStarted); - assertEquals(0, statsTracker.uploadBytesFailed); - } else { - assertEquals(0, statsTracker.totalDownloadsStarted); - assertEquals(0, statsTracker.totalDownloadsSucceeded); - assertEquals(0, statsTracker.downloadBytesStarted); - assertEquals(0, statsTracker.downloadBytesSucceeded); - } - } } From 6db95dfef7dbf8b28ad0b1e104391a9cf4a78bac Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 20 Jul 2023 11:20:48 +0530 Subject: [PATCH 25/36] Changing replica count on IT Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../java/org/opensearch/remotestore/RemoteStoreStatsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 6cbe0ea97b1d3..8eba801851d76 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -318,7 +318,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr createIndex( INDEX_NAME, Settings.builder() - .put(remoteStoreIndexSettings(dataNodeCount, 1)) + .put(remoteStoreIndexSettings(dataNodeCount - 1, 1)) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) .build() ); From 2d552b9c60cad61d4f21f733b00f433be8f290ad Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:15:09 +0530 Subject: [PATCH 26/36] Changing upper and lower bounds of random doc ingestion in IT Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../java/org/opensearch/remotestore/RemoteStoreStatsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 8eba801851d76..65a8450e70644 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -573,7 +573,7 @@ private void indexDocs() { } else { refresh(INDEX_NAME); } - int numberOfOperations = randomIntBetween(20, 50); + int numberOfOperations = randomIntBetween(10, 30); for (int j = 0; j < numberOfOperations; j++) { indexSingleDoc(INDEX_NAME); } From edbfde323b73d28a6a84c475403cfcd8c3004849 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:53:03 +0530 Subject: [PATCH 27/36] Addressing comments Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 6 --- .../opensearch/index/shard/IndexShard.java | 42 +++++++++++-------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 65a8450e70644..cfb081e3fa8e5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -20,7 +20,6 @@ import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteRefreshSegmentTracker; import org.opensearch.test.OpenSearchIntegTestCase; @@ -42,11 +41,6 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build(); - } - @Before public void setup() { setupRepo(); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index a1d1bc97ade60..8cd4176eb49e8 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4780,9 +4780,13 @@ private String copySegmentFiles( RemoteRefreshSegmentTracker downloadStatsTracker, boolean overrideLocal ) throws IOException { + boolean isRemoteStoreEnabled = indexSettings.isRemoteStoreEnabled(); + assert !isRemoteStoreEnabled || downloadStatsTracker != null + : "Download Stats tracker should not be null for Remote Store backed indices"; List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); List segmentsToDownload = new ArrayList<>(); + long sizeOfDownloadedSegments = 0; String segmentNFile = null; try { Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); @@ -4791,49 +4795,53 @@ private String copySegmentFiles( storeDirectory.deleteFile(file); } } + // Compute the number of files to download + // Required so that we don't publish metrics if there are no files to copy over for (String file : uploadedSegments.keySet()) { long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { segmentsToDownload.add(file); - // Adding size of the file to attempted download stats - if (indexSettings.isRemoteStoreEnabled()) { - downloadStatsTracker.addDownloadBytesStarted(uploadedSegments.get(file).getLength()); - } } else { skippedSegments.add(file); } } // Upload stats before starting Segment downloads // Adding a check to prevent stats being published for NoOp runs - if (!segmentsToDownload.isEmpty() && indexSettings.isRemoteStoreEnabled()) { + if (!segmentsToDownload.isEmpty() && isRemoteStoreEnabled) { downloadStatsTracker.incrementTotalDownloadsStarted(); } - long sizeOfDownloadedSegments = 0; long startTimeInMs = System.currentTimeMillis(); // Copying segments files to local store directory from remote store directory (Shard Recovery) for (String file : segmentsToDownload) { long fileSize = uploadedSegments.get(file).getLength(); + boolean fileDownloadComplete = false; + // Download bytes stats population: + // - Increments attempted bytes just before invoking copyFrom method + // - Increment succeeded bytes after copyFrom method executes successfully + // - Increment failed bytes when copyFrom method fails try { - // Throws IOException if download fails + // Adding file size to the amount of bytes attempted for download + if (isRemoteStoreEnabled) { + downloadStatsTracker.addDownloadBytesStarted(fileSize); + } storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); + fileDownloadComplete = true; downloadedSegments.add(file); - if (indexSettings.isRemoteStoreEnabled()) { - sizeOfDownloadedSegments += fileSize; - // Adding size of the file to completed download stats + sizeOfDownloadedSegments += fileSize; + // Adding file size to the amount of bytes completed in download + if (isRemoteStoreEnabled) { downloadStatsTracker.addDownloadBytesSucceeded(fileSize); } - } catch (IOException e) { - if (indexSettings.isRemoteStoreEnabled()) { - // Adding size of the file to failed download stats + } finally { + // Increment download failure stats if the `copyFrom` method above throws an exception + if (!fileDownloadComplete && isRemoteStoreEnabled) { downloadStatsTracker.addDownloadBytesFailed(fileSize); - // Incrementing overall remote sync failure counter downloadStatsTracker.incrementTotalDownloadsFailed(); } - throw e; } } - // Upload stats after download batch is completed - if (!downloadedSegments.isEmpty() && indexSettings.isRemoteStoreEnabled()) { + // Update stats after download batch is completed + if (!segmentsToDownload.isEmpty() && isRemoteStoreEnabled) { afterSyncCompleted(downloadStatsTracker, sizeOfDownloadedSegments, startTimeInMs); } // Copying segment files over to target remote directory from local store directory (Snapshot) From c1b447b24bbc67caa70eec14f7558ea4e1af815a Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:41:43 +0530 Subject: [PATCH 28/36] Addressing comments Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../RemoteStoreBackpressureIT.java | 8 +- .../remotestore/RemoteStoreStatsIT.java | 33 ++-- .../remotestore/stats/RemoteStoreStats.java | 131 ++++++++-------- .../TransportRemoteStoreStatsAction.java | 8 +- .../common/util/SegmentDownloadListener.java | 21 +++ .../RemoteRefreshSegmentPressureService.java | 48 +++--- ...java => RemoteSegmentTransferTracker.java} | 8 +- .../opensearch/index/shard/IndexShard.java | 144 +++++++++++------- .../shard/RemoteStoreRefreshListener.java | 6 +- .../stats/RemoteStoreStatsResponseTests.java | 12 +- .../stats/RemoteStoreStatsTestHelper.java | 16 +- .../stats/RemoteStoreStatsTests.java | 20 +-- .../TransportRemoteStoreStatsActionTests.java | 4 +- ...oteRefreshSegmentPressureServiceTests.java | 2 +- ...=> RemoteSegmentTransferTrackerTests.java} | 84 +++++----- .../RemoteStoreRefreshListenerTests.java | 14 +- 16 files changed, 315 insertions(+), 244 deletions(-) create mode 100644 server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java rename server/src/main/java/org/opensearch/index/remote/{RemoteRefreshSegmentTracker.java => RemoteSegmentTransferTracker.java} (99%) rename server/src/test/java/org/opensearch/index/remote/{RemoteRefreshSegmentTrackerTests.java => RemoteSegmentTransferTrackerTests.java} (92%) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java index acd1849197ca5..a9f5314173770 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java @@ -17,7 +17,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.repositories.RepositoriesService; import org.opensearch.snapshots.mockstore.MockRepository; import org.opensearch.test.OpenSearchIntegTestCase; @@ -92,7 +92,7 @@ private void validateBackpressure( assertTrue(ex.getMessage().contains("rejected execution on primary shard")); assertTrue(ex.getMessage().contains(breachMode)); - RemoteRefreshSegmentTracker.Stats stats = stats(); + RemoteSegmentTransferTracker.Stats stats = stats(); assertTrue(stats.bytesLag > 0); assertTrue(stats.refreshTimeLagMs > 0); assertTrue(stats.localRefreshNumber - stats.remoteRefreshNumber > 0); @@ -102,7 +102,7 @@ private void validateBackpressure( .setRandomControlIOExceptionRate(0d); assertBusy(() -> { - RemoteRefreshSegmentTracker.Stats finalStats = stats(); + RemoteSegmentTransferTracker.Stats finalStats = stats(); assertEquals(0, finalStats.bytesLag); assertEquals(0, finalStats.refreshTimeLagMs); assertEquals(0, finalStats.localRefreshNumber - finalStats.remoteRefreshNumber); @@ -115,7 +115,7 @@ private void validateBackpressure( deleteRepo(); } - private RemoteRefreshSegmentTracker.Stats stats() { + private RemoteSegmentTransferTracker.Stats stats() { String shardId = "0"; RemoteStoreStatsResponse response = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index cfb081e3fa8e5..66991d20ce3dd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -21,7 +21,7 @@ import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; @@ -67,7 +67,6 @@ public void testStatsResponseFromAllNodes() { String shardId = "0"; for (String node : nodes) { RemoteStoreStatsResponse response = client(node).admin().cluster().prepareRemoteStoreStats(INDEX_NAME, shardId).get(); - logger.info("Stats Response : " + response); assertTrue(response.getSuccessfulShards() > 0); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length != 0); final String indexShardId = String.format(Locale.ROOT, "[%s][%s]", INDEX_NAME, shardId); @@ -75,7 +74,7 @@ public void testStatsResponseFromAllNodes() { .filter(stat -> indexShardId.equals(stat.getStats().shardId.toString())) .collect(Collectors.toList()); assertEquals(1, matches.size()); - RemoteRefreshSegmentTracker.Stats stats = matches.get(0).getStats(); + RemoteSegmentTransferTracker.Stats stats = matches.get(0).getStats(); validateUploadStats(stats); assertEquals(0, stats.totalDownloadsStarted); } @@ -95,7 +94,7 @@ public void testStatsResponseFromAllNodes() { for (RemoteStoreStats stat : matches) { ShardRouting routing = stat.getShardRouting(); validateShardRouting(routing); - RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); + RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); assertEquals(0, stats.totalDownloadsStarted); @@ -127,7 +126,7 @@ public void testStatsResponseAllShards() { RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); assertEquals(3, response.getSuccessfulShards()); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 3); - RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); + RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); assertEquals(0, stats.totalDownloadsStarted); @@ -174,7 +173,7 @@ public void testStatsResponseFromLocalNode() { RemoteStoreStatsResponse response = remoteStoreStatsRequestBuilder.get(); assertEquals(1, response.getSuccessfulShards()); assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 1); - RemoteRefreshSegmentTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); + RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); assertEquals(0, stats.totalDownloadsStarted); } @@ -190,7 +189,7 @@ public void testStatsResponseFromLocalNode() { for (RemoteStoreStats stat : response.getRemoteStoreStats()) { ShardRouting routing = stat.getShardRouting(); validateShardRouting(routing); - RemoteRefreshSegmentTracker.Stats stats = stat.getStats(); + RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); assertEquals(0, stats.totalDownloadsStarted); @@ -221,7 +220,7 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce // Get zero state values // Extract and assert zero state primary stats RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - RemoteRefreshSegmentTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + RemoteSegmentTransferTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()) .get(0) @@ -237,7 +236,7 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce assertTrue(zeroStatePrimaryStats.totalUploadsFailed == 0 && zeroStatePrimaryStats.uploadBytesFailed == 0); // Extract and assert zero state replica stats - RemoteRefreshSegmentTracker.Stats zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + RemoteSegmentTransferTracker.Stats zeroStateReplicaStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()) .get(0) @@ -268,8 +267,8 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()); assertEquals(1, replicaStatsList.size()); - RemoteRefreshSegmentTracker.Stats primaryStats = primaryStatsList.get(0).getStats(); - RemoteRefreshSegmentTracker.Stats replicaStats = replicaStatsList.get(0).getStats(); + RemoteSegmentTransferTracker.Stats primaryStats = primaryStatsList.get(0).getStats(); + RemoteSegmentTransferTracker.Stats replicaStats = replicaStatsList.get(0).getStats(); // Assert Upload syncs - zero state uploads == download syncs assertTrue( primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted - 1 == replicaStats.totalDownloadsStarted @@ -324,7 +323,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr // Get zero state values // Extract and assert zero state primary stats RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); - RemoteRefreshSegmentTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) + RemoteSegmentTransferTracker.Stats zeroStatePrimaryStats = Arrays.stream(zeroStateResponse.getRemoteStoreStats()) .filter(remoteStoreStats -> remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()) .get(0) @@ -372,7 +371,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr // Assert that stats for primary shard and replica shard set are equal for (RemoteStoreStats eachStatsObject : response.getRemoteStoreStats()) { - RemoteRefreshSegmentTracker.Stats stats = eachStatsObject.getStats(); + RemoteSegmentTransferTracker.Stats stats = eachStatsObject.getStats(); if (eachStatsObject.getShardRouting().primary()) { uploadsStarted = stats.totalUploadsStarted; uploadsSucceeded = stats.totalUploadsSucceeded; @@ -513,7 +512,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { .get() .getRemoteStoreStats(); Arrays.stream(remoteStoreStats).forEach(statObject -> { - RemoteRefreshSegmentTracker.Stats segmentTracker = statObject.getStats(); + RemoteSegmentTransferTracker.Stats segmentTracker = statObject.getStats(); // Assert that we have both upload and download stats for the index assertTrue( segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0 @@ -542,7 +541,7 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce .get() .getRemoteStoreStats(); Arrays.stream(remoteStoreStats).forEach(statObject -> { - RemoteRefreshSegmentTracker.Stats segmentTracker = statObject.getStats(); + RemoteSegmentTransferTracker.Stats segmentTracker = statObject.getStats(); if (statObject.getShardRouting().primary()) { assertTrue( segmentTracker.totalUploadsSucceeded == 1 @@ -590,7 +589,7 @@ private void relocateShard(int shardId, String sourceNode, String destNode) { ensureGreen(INDEX_NAME); } - private void validateUploadStats(RemoteRefreshSegmentTracker.Stats stats) { + private void validateUploadStats(RemoteSegmentTransferTracker.Stats stats) { assertEquals(0, stats.refreshTimeLagMs); assertEquals(stats.localRefreshNumber, stats.remoteRefreshNumber); assertTrue(stats.uploadBytesStarted > 0); @@ -607,7 +606,7 @@ private void validateUploadStats(RemoteRefreshSegmentTracker.Stats stats) { assertTrue(stats.uploadTimeMovingAverage > 0); } - private void validateDownloadStats(RemoteRefreshSegmentTracker.Stats stats) { + private void validateDownloadStats(RemoteSegmentTransferTracker.Stats stats) { assertTrue(stats.lastDownloadTimestampMs > 0); assertTrue(stats.totalDownloadsStarted > 0); assertTrue(stats.totalDownloadsSucceeded > 0); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index 9c732ce2c3161..ca92e8fd082df 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -14,7 +14,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import java.io.IOException; @@ -25,21 +25,21 @@ */ public class RemoteStoreStats implements Writeable, ToXContentFragment { - private final RemoteRefreshSegmentTracker.Stats remoteSegmentShardStats; + private final RemoteSegmentTransferTracker.Stats remoteSegmentShardStats; private final ShardRouting shardRouting; - public RemoteStoreStats(RemoteRefreshSegmentTracker.Stats remoteSegmentUploadShardStats, ShardRouting shardRouting) { + public RemoteStoreStats(RemoteSegmentTransferTracker.Stats remoteSegmentUploadShardStats, ShardRouting shardRouting) { this.remoteSegmentShardStats = remoteSegmentUploadShardStats; this.shardRouting = shardRouting; } public RemoteStoreStats(StreamInput in) throws IOException { - this.remoteSegmentShardStats = in.readOptionalWriteable(RemoteRefreshSegmentTracker.Stats::new); + this.remoteSegmentShardStats = in.readOptionalWriteable(RemoteSegmentTransferTracker.Stats::new); this.shardRouting = new ShardRouting(in); } - public RemoteRefreshSegmentTracker.Stats getStats() { + public RemoteSegmentTransferTracker.Stats getStats() { return remoteSegmentShardStats; } @@ -50,71 +50,18 @@ public ShardRouting getShardRouting() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.startObject(Fields.ROUTING); - builder.field(RoutingFields.STATE, shardRouting.state()); - builder.field(RoutingFields.PRIMARY, shardRouting.primary()); - builder.field(RoutingFields.NODE_ID, shardRouting.currentNodeId()); - builder.endObject(); + buildShardRouting(builder); builder.startObject(Fields.SEGMENT); builder.startObject(SubFields.DOWNLOAD); // Ensuring that we are not showing 0 metrics to the user if (remoteSegmentShardStats.totalDownloadsStarted != 0) { - builder.field(DownloadStatsFields.LAST_SYNC_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); - builder.startObject(DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE) - .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); - builder.endObject(); - builder.startObject(DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES) - .field(SubFields.STARTED, remoteSegmentShardStats.downloadBytesStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.downloadBytesSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.downloadBytesFailed); - builder.endObject(); - builder.startObject(DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES) - .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulSegmentDownloadBytes) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesMovingAverage); - builder.endObject(); - builder.startObject(DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesPerSecMovingAverage); - builder.endObject(); - builder.startObject(DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadTimeMovingAverage); - builder.endObject(); + buildDownloadStats(builder); } builder.endObject(); builder.startObject(SubFields.UPLOAD); // Ensuring that we are not showing 0 metrics to the user if (remoteSegmentShardStats.totalUploadsStarted != 0) { - builder.field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) - .field(UploadStatsFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) - .field(UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) - .field( - UploadStatsFields.REFRESH_LAG, - remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber - ) - .field(UploadStatsFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) - .field(UploadStatsFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) - .field(UploadStatsFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); - builder.startObject(UploadStatsFields.TOTAL_SYNCS_TO_REMOTE) - .field(SubFields.STARTED, remoteSegmentShardStats.totalUploadsStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalUploadsSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.totalUploadsFailed); - builder.endObject(); - builder.startObject(UploadStatsFields.TOTAL_UPLOADS_IN_BYTES) - .field(SubFields.STARTED, remoteSegmentShardStats.uploadBytesStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.uploadBytesSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.uploadBytesFailed); - builder.endObject(); - builder.startObject(UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES) - .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulRemoteRefreshBytes) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesMovingAverage); - builder.endObject(); - builder.startObject(UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesPerSecMovingAverage); - builder.endObject(); - builder.startObject(UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadTimeMovingAverage); - builder.endObject(); + buildUploadStats(builder); } builder.endObject(); builder.endObject(); @@ -127,6 +74,68 @@ public void writeTo(StreamOutput out) throws IOException { shardRouting.writeTo(out); } + private void buildUploadStats(XContentBuilder builder) throws IOException { + builder.field(UploadStatsFields.LOCAL_REFRESH_TIMESTAMP, remoteSegmentShardStats.localRefreshClockTimeMs) + .field(UploadStatsFields.REMOTE_REFRESH_TIMESTAMP, remoteSegmentShardStats.remoteRefreshClockTimeMs) + .field(UploadStatsFields.REFRESH_TIME_LAG_IN_MILLIS, remoteSegmentShardStats.refreshTimeLagMs) + .field(UploadStatsFields.REFRESH_LAG, remoteSegmentShardStats.localRefreshNumber - remoteSegmentShardStats.remoteRefreshNumber) + .field(UploadStatsFields.BYTES_LAG, remoteSegmentShardStats.bytesLag) + .field(UploadStatsFields.BACKPRESSURE_REJECTION_COUNT, remoteSegmentShardStats.rejectionCount) + .field(UploadStatsFields.CONSECUTIVE_FAILURE_COUNT, remoteSegmentShardStats.consecutiveFailuresCount); + builder.startObject(UploadStatsFields.TOTAL_SYNCS_TO_REMOTE) + .field(SubFields.STARTED, remoteSegmentShardStats.totalUploadsStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalUploadsSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.totalUploadsFailed); + builder.endObject(); + builder.startObject(UploadStatsFields.TOTAL_UPLOADS_IN_BYTES) + .field(SubFields.STARTED, remoteSegmentShardStats.uploadBytesStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.uploadBytesSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.uploadBytesFailed); + builder.endObject(); + builder.startObject(UploadStatsFields.REMOTE_REFRESH_SIZE_IN_BYTES) + .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulRemoteRefreshBytes) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesMovingAverage); + builder.endObject(); + builder.startObject(UploadStatsFields.UPLOAD_LATENCY_IN_BYTES_PER_SEC) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadBytesPerSecMovingAverage); + builder.endObject(); + builder.startObject(UploadStatsFields.REMOTE_REFRESH_LATENCY_IN_MILLIS) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.uploadTimeMovingAverage); + builder.endObject(); + } + + private void buildDownloadStats(XContentBuilder builder) throws IOException { + builder.field(DownloadStatsFields.LAST_SYNC_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); + builder.startObject(DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE) + .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); + builder.endObject(); + builder.startObject(DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES) + .field(SubFields.STARTED, remoteSegmentShardStats.downloadBytesStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.downloadBytesSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.downloadBytesFailed); + builder.endObject(); + builder.startObject(DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES) + .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulSegmentDownloadBytes) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesMovingAverage); + builder.endObject(); + builder.startObject(DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesPerSecMovingAverage); + builder.endObject(); + builder.startObject(DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadTimeMovingAverage); + builder.endObject(); + } + + private void buildShardRouting(XContentBuilder builder) throws IOException { + builder.startObject(Fields.ROUTING); + builder.field(RoutingFields.STATE, shardRouting.state()); + builder.field(RoutingFields.PRIMARY, shardRouting.primary()); + builder.field(RoutingFields.NODE_ID, shardRouting.currentNodeId()); + builder.endObject(); + } + static final class Fields { static final String ROUTING = "routing"; static final String SEGMENT = "segment"; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java index 827b403504bc4..37835a5add3d6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsAction.java @@ -24,7 +24,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.index.IndexService; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardNotFoundException; import org.opensearch.indices.IndicesService; @@ -153,10 +153,10 @@ protected RemoteStoreStats shardOperation(RemoteStoreStatsRequest request, Shard throw new ShardNotFoundException(indexShard.shardId()); } - RemoteRefreshSegmentTracker remoteRefreshSegmentTracker = remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker( + RemoteSegmentTransferTracker remoteSegmentTransferTracker = remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker( indexShard.shardId() ); - assert Objects.nonNull(remoteRefreshSegmentTracker); - return new RemoteStoreStats(remoteRefreshSegmentTracker.stats(), indexShard.routingEntry()); + assert Objects.nonNull(remoteSegmentTransferTracker); + return new RemoteStoreStats(remoteSegmentTransferTracker.stats(), indexShard.routingEntry()); } } diff --git a/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java b/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java new file mode 100644 index 0000000000000..49af6a34891fe --- /dev/null +++ b/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.util; + +public interface SegmentDownloadListener { + void beforeSync(); + + void afterSync(long downloadedFilesSize, long startTimeInMs); + + void beforeFileDownload(long fileSize); + + void afterFileDownload(long fileSize); + + void fileDownloadFailed(long fileSize); +} diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java index 3f1161f0c5e03..75f22054a4c42 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java @@ -26,7 +26,7 @@ import java.util.function.BiConsumer; /** - * Service used to validate if the incoming indexing request should be rejected based on the {@link RemoteRefreshSegmentTracker}. + * Service used to validate if the incoming indexing request should be rejected based on the {@link RemoteSegmentTransferTracker}. * * @opensearch.internal */ @@ -37,7 +37,7 @@ public class RemoteRefreshSegmentPressureService implements IndexEventListener { /** * Keeps map of remote-backed index shards and their corresponding backpressure tracker. */ - private final Map trackerMap = ConcurrentCollections.newConcurrentMap(); + private final Map trackerMap = ConcurrentCollections.newConcurrentMap(); /** * Remote refresh segment pressure settings which is used for creation of the backpressure tracker and as well as rejection. @@ -57,12 +57,12 @@ public RemoteRefreshSegmentPressureService(ClusterService clusterService, Settin } /** - * Get {@code RemoteRefreshSegmentTracker} only if the underlying Index has remote segments integration enabled. + * Get {@code RemoteSegmentTransferTracker} only if the underlying Index has remote segments integration enabled. * * @param shardId shard id * @return the tracker if index is remote-backed, else null. */ - public RemoteRefreshSegmentTracker getRemoteRefreshSegmentTracker(ShardId shardId) { + public RemoteSegmentTransferTracker getRemoteRefreshSegmentTracker(ShardId shardId) { return trackerMap.get(shardId); } @@ -74,7 +74,7 @@ public void afterIndexShardCreated(IndexShard indexShard) { ShardId shardId = indexShard.shardId(); trackerMap.put( shardId, - new RemoteRefreshSegmentTracker( + new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -86,8 +86,8 @@ public void afterIndexShardCreated(IndexShard indexShard) { @Override public void afterIndexShardClosed(ShardId shardId, IndexShard indexShard, Settings indexSettings) { - RemoteRefreshSegmentTracker remoteRefreshSegmentTracker = trackerMap.remove(shardId); - if (remoteRefreshSegmentTracker != null) { + RemoteSegmentTransferTracker remoteSegmentTransferTracker = trackerMap.remove(shardId); + if (remoteSegmentTransferTracker != null) { logger.trace("Deleted tracker for shardId={}", shardId); } } @@ -107,34 +107,34 @@ public boolean isSegmentsUploadBackpressureEnabled() { * @param shardId shardId for which the validation needs to be done. */ public void validateSegmentsUploadLag(ShardId shardId) { - RemoteRefreshSegmentTracker remoteRefreshSegmentTracker = getRemoteRefreshSegmentTracker(shardId); + RemoteSegmentTransferTracker remoteSegmentTransferTracker = getRemoteRefreshSegmentTracker(shardId); // condition 1 - This will be null for non-remote backed indexes // condition 2 - This will be zero if the remote store is - if (remoteRefreshSegmentTracker == null || remoteRefreshSegmentTracker.getRefreshSeqNoLag() == 0) { + if (remoteSegmentTransferTracker == null || remoteSegmentTransferTracker.getRefreshSeqNoLag() == 0) { return; } for (LagValidator lagValidator : lagValidators) { - if (lagValidator.validate(remoteRefreshSegmentTracker, shardId) == false) { - remoteRefreshSegmentTracker.incrementRejectionCount(lagValidator.name()); - throw new OpenSearchRejectedExecutionException(lagValidator.rejectionMessage(remoteRefreshSegmentTracker, shardId)); + if (lagValidator.validate(remoteSegmentTransferTracker, shardId) == false) { + remoteSegmentTransferTracker.incrementRejectionCount(lagValidator.name()); + throw new OpenSearchRejectedExecutionException(lagValidator.rejectionMessage(remoteSegmentTransferTracker, shardId)); } } } void updateUploadBytesMovingAverageWindowSize(int updatedSize) { - updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateUploadBytesMovingAverageWindowSize, updatedSize); + updateMovingAverageWindowSize(RemoteSegmentTransferTracker::updateUploadBytesMovingAverageWindowSize, updatedSize); } void updateUploadBytesPerSecMovingAverageWindowSize(int updatedSize) { - updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateUploadBytesPerSecMovingAverageWindowSize, updatedSize); + updateMovingAverageWindowSize(RemoteSegmentTransferTracker::updateUploadBytesPerSecMovingAverageWindowSize, updatedSize); } void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { - updateMovingAverageWindowSize(RemoteRefreshSegmentTracker::updateUploadTimeMsMovingAverageWindowSize, updatedSize); + updateMovingAverageWindowSize(RemoteSegmentTransferTracker::updateUploadTimeMsMovingAverageWindowSize, updatedSize); } - void updateMovingAverageWindowSize(BiConsumer biConsumer, int updatedSize) { + void updateMovingAverageWindowSize(BiConsumer biConsumer, int updatedSize) { trackerMap.values().forEach(tracker -> biConsumer.accept(tracker, updatedSize)); } @@ -158,7 +158,7 @@ private LagValidator(RemoteRefreshSegmentPressureSettings pressureSettings) { * @param shardId shard id of the {@code IndexShard} currently being validated. * @return true if successfully validated that lag is acceptable. */ - abstract boolean validate(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId); + abstract boolean validate(RemoteSegmentTransferTracker pressureTracker, ShardId shardId); /** * Returns the name of the lag validator. @@ -167,7 +167,7 @@ private LagValidator(RemoteRefreshSegmentPressureSettings pressureSettings) { */ abstract String name(); - abstract String rejectionMessage(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId); + abstract String rejectionMessage(RemoteSegmentTransferTracker pressureTracker, ShardId shardId); } /** @@ -184,7 +184,7 @@ private BytesLagValidator(RemoteRefreshSegmentPressureSettings pressureSettings) } @Override - public boolean validate(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId) { + public boolean validate(RemoteSegmentTransferTracker pressureTracker, ShardId shardId) { if (pressureTracker.getRefreshSeqNoLag() <= 1) { return true; } @@ -198,7 +198,7 @@ public boolean validate(RemoteRefreshSegmentTracker pressureTracker, ShardId sha } @Override - public String rejectionMessage(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId) { + public String rejectionMessage(RemoteSegmentTransferTracker pressureTracker, ShardId shardId) { double dynamicBytesLagThreshold = pressureTracker.getUploadBytesAverage() * pressureSettings.getBytesLagVarianceFactor(); return String.format( Locale.ROOT, @@ -230,7 +230,7 @@ private TimeLagValidator(RemoteRefreshSegmentPressureSettings pressureSettings) } @Override - public boolean validate(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId) { + public boolean validate(RemoteSegmentTransferTracker pressureTracker, ShardId shardId) { if (pressureTracker.getRefreshSeqNoLag() <= 1) { return true; } @@ -244,7 +244,7 @@ public boolean validate(RemoteRefreshSegmentTracker pressureTracker, ShardId sha } @Override - public String rejectionMessage(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId) { + public String rejectionMessage(RemoteSegmentTransferTracker pressureTracker, ShardId shardId) { double dynamicTimeLagThreshold = pressureTracker.getUploadTimeMsAverage() * pressureSettings.getUploadTimeLagVarianceFactor(); return String.format( Locale.ROOT, @@ -276,14 +276,14 @@ private ConsecutiveFailureValidator(RemoteRefreshSegmentPressureSettings pressur } @Override - public boolean validate(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId) { + public boolean validate(RemoteSegmentTransferTracker pressureTracker, ShardId shardId) { int failureStreakCount = pressureTracker.getConsecutiveFailureCount(); int minConsecutiveFailureThreshold = pressureSettings.getMinConsecutiveFailuresLimit(); return failureStreakCount <= minConsecutiveFailureThreshold; } @Override - public String rejectionMessage(RemoteRefreshSegmentTracker pressureTracker, ShardId shardId) { + public String rejectionMessage(RemoteSegmentTransferTracker pressureTracker, ShardId shardId) { return String.format( Locale.ROOT, "rejected execution on primary shard:%s due to remote segments lagging behind local segments." diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java similarity index 99% rename from server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java rename to server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java index 72342927d9316..9f68509049a79 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java @@ -30,7 +30,7 @@ * * @opensearch.internal */ -public class RemoteRefreshSegmentTracker { +public class RemoteSegmentTransferTracker { /** * ShardId for which this instance tracks the remote segment upload metadata. @@ -225,7 +225,7 @@ public class RemoteRefreshSegmentTracker { private final int SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE = 20; - public RemoteRefreshSegmentTracker( + public RemoteSegmentTransferTracker( ShardId shardId, int uploadBytesMovingAverageWindowSize, int uploadBytesPerSecMovingAverageWindowSize, @@ -631,8 +631,8 @@ public void addDownloadTime(long timeMs) { this.downloadTimeMovingAverageReference.record(timeMs); } - public RemoteRefreshSegmentTracker.Stats stats() { - return new RemoteRefreshSegmentTracker.Stats( + public RemoteSegmentTransferTracker.Stats stats() { + return new RemoteSegmentTransferTracker.Stats( shardId, localRefreshClockTimeMs, remoteRefreshClockTimeMs, diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 8cd4176eb49e8..de50bef3daeb5 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -58,6 +58,7 @@ import org.apache.lucene.util.ThreadInterruptedException; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.cluster.metadata.DataStream; +import org.opensearch.common.util.SegmentDownloadListener; import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; @@ -146,7 +147,7 @@ import org.opensearch.index.recovery.RecoveryStats; import org.opensearch.index.refresh.RefreshStats; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.search.stats.ShardSearchStats; import org.opensearch.index.seqno.ReplicationTracker; @@ -740,6 +741,7 @@ public void onFailure(Exception e) { + this.shardRouting; shardStateUpdated.countDown(); } + if (currentRouting.active() == false && newRouting.active()) { indexEventListener.afterIndexShardStarted(this); } @@ -4648,14 +4650,7 @@ public void syncSegmentsFromRemoteSegmentStore(boolean overrideLocal, boolean re storeDirectory = store.directory(); } Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); - copySegmentFiles( - storeDirectory, - remoteDirectory, - null, - uploadedSegments, - remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), - overrideLocal - ); + copySegmentFiles(storeDirectory, remoteDirectory, null, uploadedSegments, overrideLocal); if (refreshLevelSegmentSync && remoteSegmentMetadata != null) { try ( @@ -4742,7 +4737,6 @@ public void syncSegmentsFromGivenRemoteSegmentStore( sourceRemoteDirectory, remoteDirectory, uploadedSegments, - remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), overrideLocal ); if (segmentsNFile != null) { @@ -4777,12 +4771,12 @@ private String copySegmentFiles( RemoteSegmentStoreDirectory sourceRemoteDirectory, RemoteSegmentStoreDirectory targetRemoteDirectory, Map uploadedSegments, - RemoteRefreshSegmentTracker downloadStatsTracker, boolean overrideLocal ) throws IOException { boolean isRemoteStoreEnabled = indexSettings.isRemoteStoreEnabled(); - assert !isRemoteStoreEnabled || downloadStatsTracker != null - : "Download Stats tracker should not be null for Remote Store backed indices"; + SegmentDownloadListener segmentDownloadListener = initializeSegmentDownloadListener( + remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId) + ); List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); List segmentsToDownload = new ArrayList<>(); @@ -4808,41 +4802,25 @@ private String copySegmentFiles( // Upload stats before starting Segment downloads // Adding a check to prevent stats being published for NoOp runs if (!segmentsToDownload.isEmpty() && isRemoteStoreEnabled) { - downloadStatsTracker.incrementTotalDownloadsStarted(); + segmentDownloadListener.beforeSync(); } long startTimeInMs = System.currentTimeMillis(); // Copying segments files to local store directory from remote store directory (Shard Recovery) for (String file : segmentsToDownload) { - long fileSize = uploadedSegments.get(file).getLength(); - boolean fileDownloadComplete = false; - // Download bytes stats population: - // - Increments attempted bytes just before invoking copyFrom method - // - Increment succeeded bytes after copyFrom method executes successfully - // - Increment failed bytes when copyFrom method fails - try { - // Adding file size to the amount of bytes attempted for download - if (isRemoteStoreEnabled) { - downloadStatsTracker.addDownloadBytesStarted(fileSize); - } - storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - fileDownloadComplete = true; - downloadedSegments.add(file); - sizeOfDownloadedSegments += fileSize; - // Adding file size to the amount of bytes completed in download - if (isRemoteStoreEnabled) { - downloadStatsTracker.addDownloadBytesSucceeded(fileSize); - } - } finally { - // Increment download failure stats if the `copyFrom` method above throws an exception - if (!fileDownloadComplete && isRemoteStoreEnabled) { - downloadStatsTracker.addDownloadBytesFailed(fileSize); - downloadStatsTracker.incrementTotalDownloadsFailed(); - } - } + sizeOfDownloadedSegments = statsAwareCopyFromRemoteStore( + storeDirectory, + sourceRemoteDirectory, + uploadedSegments, + segmentDownloadListener, + isRemoteStoreEnabled, + downloadedSegments, + sizeOfDownloadedSegments, + file + ); } // Update stats after download batch is completed if (!segmentsToDownload.isEmpty() && isRemoteStoreEnabled) { - afterSyncCompleted(downloadStatsTracker, sizeOfDownloadedSegments, startTimeInMs); + segmentDownloadListener.afterSync(sizeOfDownloadedSegments, startTimeInMs); } // Copying segment files over to target remote directory from local store directory (Snapshot) for (String file : uploadedSegments.keySet()) { @@ -4861,6 +4839,80 @@ private String copySegmentFiles( return segmentNFile; } + private SegmentDownloadListener initializeSegmentDownloadListener(RemoteSegmentTransferTracker downloadStatsTracker) { + return new SegmentDownloadListener() { + @Override + public void beforeSync() { + downloadStatsTracker.incrementTotalDownloadsStarted(); + } + + @Override + public void afterSync(long downloadedFilesSize, long startTimeInMs) { + long currentTimeInMs = System.currentTimeMillis(); + downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); + downloadStatsTracker.incrementTotalDownloadsSucceeded(); + downloadStatsTracker.addDownloadBytes(downloadedFilesSize); + long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); + downloadStatsTracker.addDownloadTime(timeTakenInMS); + downloadStatsTracker.addDownloadBytesPerSec((downloadedFilesSize * 1_000L) / timeTakenInMS); + } + + @Override + public void beforeFileDownload(long fileSize) { + downloadStatsTracker.addDownloadBytesStarted(fileSize); + } + + @Override + public void afterFileDownload(long fileSize) { + downloadStatsTracker.addDownloadBytesSucceeded(fileSize); + } + + @Override + public void fileDownloadFailed(long fileSize) { + downloadStatsTracker.addDownloadBytesFailed(fileSize); + downloadStatsTracker.incrementTotalDownloadsFailed(); + } + }; + } + + private long statsAwareCopyFromRemoteStore( + Directory storeDirectory, + RemoteSegmentStoreDirectory sourceRemoteDirectory, + Map uploadedSegments, + SegmentDownloadListener segmentDownloadListener, + boolean isRemoteStoreEnabled, + List downloadedSegments, + long sizeOfDownloadedSegments, + String file + ) throws IOException { + long fileSize = uploadedSegments.get(file).getLength(); + boolean fileDownloadComplete = false; + // Download bytes stats population: + // - Increments attempted bytes just before invoking copyFrom method + // - Increment succeeded bytes after copyFrom method executes successfully + // - Increment failed bytes when copyFrom method fails + try { + // Adding file size to the amount of bytes attempted for download + if (isRemoteStoreEnabled) { + segmentDownloadListener.beforeFileDownload(fileSize); + } + storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); + fileDownloadComplete = true; + downloadedSegments.add(file); + sizeOfDownloadedSegments += fileSize; + // Adding file size to the amount of bytes completed in download + if (isRemoteStoreEnabled) { + segmentDownloadListener.afterFileDownload(fileSize); + } + } finally { + // Increment download failure stats if the `copyFrom` method above throws an exception + if (!fileDownloadComplete && isRemoteStoreEnabled) { + segmentDownloadListener.fileDownloadFailed(fileSize); + } + } + return sizeOfDownloadedSegments; + } + private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { if (checksum == CodecUtil.retrieveChecksum(indexInput)) { @@ -4876,16 +4928,6 @@ private boolean localDirectoryContains(Directory localDirectory, String file, lo return false; } - private void afterSyncCompleted(RemoteRefreshSegmentTracker downloadStatsTracker, long downloadedFileSize, long startTimeInMs) { - long currentTimeInMs = System.currentTimeMillis(); - downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); - downloadStatsTracker.incrementTotalDownloadsSucceeded(); - downloadStatsTracker.addDownloadBytes(downloadedFileSize); - long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); - downloadStatsTracker.addDownloadTime(timeTakenInMS); - downloadStatsTracker.addDownloadBytesPerSec((downloadedFileSize * 1_000L) / timeTakenInMS); - } - /** * Returns the maximum sequence number of either update or delete operations have been processed in this shard * or the sequence number from {@link #advanceMaxSeqNoOfUpdatesOrDeletes(long)}. An index request is considered diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java index 3ea8278038ac5..fa4027f1306b6 100644 --- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java @@ -28,7 +28,7 @@ import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.index.engine.EngineException; import org.opensearch.index.engine.InternalEngine; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; @@ -87,7 +87,7 @@ public final class RemoteStoreRefreshListener extends CloseableRetryableRefreshL private final IndexShard indexShard; private final Directory storeDirectory; private final RemoteSegmentStoreDirectory remoteDirectory; - private final RemoteRefreshSegmentTracker segmentTracker; + private final RemoteSegmentTransferTracker segmentTracker; private final Map localSegmentChecksumMap; private long primaryTerm; private volatile Iterator backoffDelayIterator; @@ -104,7 +104,7 @@ public final class RemoteStoreRefreshListener extends CloseableRetryableRefreshL public RemoteStoreRefreshListener( IndexShard indexShard, SegmentReplicationCheckpointPublisher checkpointPublisher, - RemoteRefreshSegmentTracker segmentTracker + RemoteSegmentTransferTracker segmentTracker ) { super(indexShard.getThreadPool()); logger = Loggers.getLogger(getClass(), indexShard.shardId()); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java index 1a562e68f1745..64dfda86c1af9 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsResponseTests.java @@ -14,7 +14,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -48,7 +48,7 @@ public void tearDown() throws Exception { } public void testSerializationForPrimary() throws Exception { - RemoteRefreshSegmentTracker.Stats mockPrimaryTrackerStats = createStatsForNewPrimary(shardId); + RemoteSegmentTransferTracker.Stats mockPrimaryTrackerStats = createStatsForNewPrimary(shardId); ShardRouting primaryShardRouting = createShardRouting(shardId, true); RemoteStoreStats primaryShardStats = new RemoteStoreStats(mockPrimaryTrackerStats, primaryShardRouting); RemoteStoreStatsResponse statsResponse = new RemoteStoreStatsResponse( @@ -77,8 +77,8 @@ public void testSerializationForPrimary() throws Exception { } public void testSerializationForBothPrimaryAndReplica() throws Exception { - RemoteRefreshSegmentTracker.Stats mockPrimaryTrackerStats = createStatsForNewPrimary(shardId); - RemoteRefreshSegmentTracker.Stats mockReplicaTrackerStats = createStatsForNewReplica(shardId); + RemoteSegmentTransferTracker.Stats mockPrimaryTrackerStats = createStatsForNewPrimary(shardId); + RemoteSegmentTransferTracker.Stats mockReplicaTrackerStats = createStatsForNewReplica(shardId); ShardRouting primaryShardRouting = createShardRouting(shardId, true); ShardRouting replicaShardRouting = createShardRouting(shardId, false); RemoteStoreStats primaryShardStats = new RemoteStoreStats(mockPrimaryTrackerStats, primaryShardRouting); @@ -117,8 +117,8 @@ public void testSerializationForBothPrimaryAndReplica() throws Exception { } public void testSerializationForBothRemoteStoreRestoredPrimaryAndReplica() throws Exception { - RemoteRefreshSegmentTracker.Stats mockPrimaryTrackerStats = createStatsForRemoteStoreRestoredPrimary(shardId); - RemoteRefreshSegmentTracker.Stats mockReplicaTrackerStats = createStatsForNewReplica(shardId); + RemoteSegmentTransferTracker.Stats mockPrimaryTrackerStats = createStatsForRemoteStoreRestoredPrimary(shardId); + RemoteSegmentTransferTracker.Stats mockReplicaTrackerStats = createStatsForNewReplica(shardId); ShardRouting primaryShardRouting = createShardRouting(shardId, true); ShardRouting replicaShardRouting = createShardRouting(shardId, false); RemoteStoreStats primaryShardStats = new RemoteStoreStats(mockPrimaryTrackerStats, primaryShardRouting); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index c6cef9357dc23..c400126bb80a3 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -11,7 +11,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.TestShardRouting; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.core.index.shard.ShardId; import java.util.Map; @@ -24,8 +24,8 @@ * Helper utilities for Remote Store stats tests */ public class RemoteStoreStatsTestHelper { - static RemoteRefreshSegmentTracker.Stats createStatsForNewPrimary(ShardId shardId) { - return new RemoteRefreshSegmentTracker.Stats( + static RemoteSegmentTransferTracker.Stats createStatsForNewPrimary(ShardId shardId) { + return new RemoteSegmentTransferTracker.Stats( shardId, 101, 102, @@ -59,8 +59,8 @@ static RemoteRefreshSegmentTracker.Stats createStatsForNewPrimary(ShardId shardI ); } - static RemoteRefreshSegmentTracker.Stats createStatsForNewReplica(ShardId shardId) { - return new RemoteRefreshSegmentTracker.Stats( + static RemoteSegmentTransferTracker.Stats createStatsForNewReplica(ShardId shardId) { + return new RemoteSegmentTransferTracker.Stats( shardId, 0, 0, @@ -94,8 +94,8 @@ static RemoteRefreshSegmentTracker.Stats createStatsForNewReplica(ShardId shardI ); } - static RemoteRefreshSegmentTracker.Stats createStatsForRemoteStoreRestoredPrimary(ShardId shardId) { - return new RemoteRefreshSegmentTracker.Stats( + static RemoteSegmentTransferTracker.Stats createStatsForRemoteStoreRestoredPrimary(ShardId shardId) { + return new RemoteSegmentTransferTracker.Stats( shardId, 50, 50, @@ -135,7 +135,7 @@ static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { static void compareStatsResponse( Map statsObject, - RemoteRefreshSegmentTracker.Stats statsTracker, + RemoteSegmentTransferTracker.Stats statsTracker, ShardRouting routing ) { assertEquals( diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 3ba9733e1800b..b2193fd658660 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -15,7 +15,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.core.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -49,7 +49,7 @@ public void tearDown() throws Exception { } public void testXContentBuilderWithPrimaryShard() throws IOException { - RemoteRefreshSegmentTracker.Stats uploadStats = createStatsForNewPrimary(shardId); + RemoteSegmentTransferTracker.Stats uploadStats = createStatsForNewPrimary(shardId); ShardRouting routing = createShardRouting(shardId, true); RemoteStoreStats stats = new RemoteStoreStats(uploadStats, routing); @@ -60,7 +60,7 @@ public void testXContentBuilderWithPrimaryShard() throws IOException { } public void testXContentBuilderWithReplicaShard() throws IOException { - RemoteRefreshSegmentTracker.Stats downloadStats = createStatsForNewReplica(shardId); + RemoteSegmentTransferTracker.Stats downloadStats = createStatsForNewReplica(shardId); ShardRouting routing = createShardRouting(shardId, false); RemoteStoreStats stats = new RemoteStoreStats(downloadStats, routing); @@ -71,7 +71,7 @@ public void testXContentBuilderWithReplicaShard() throws IOException { } public void testXContentBuilderWithRemoteStoreRestoredShard() throws IOException { - RemoteRefreshSegmentTracker.Stats remotestoreRestoredShardStats = createStatsForRemoteStoreRestoredPrimary(shardId); + RemoteSegmentTransferTracker.Stats remotestoreRestoredShardStats = createStatsForRemoteStoreRestoredPrimary(shardId); ShardRouting routing = createShardRouting(shardId, true); RemoteStoreStats stats = new RemoteStoreStats(remotestoreRestoredShardStats, routing); @@ -82,12 +82,12 @@ public void testXContentBuilderWithRemoteStoreRestoredShard() throws IOException } public void testSerializationForPrimaryShard() throws Exception { - RemoteRefreshSegmentTracker.Stats primaryShardStats = createStatsForNewPrimary(shardId); + RemoteSegmentTransferTracker.Stats primaryShardStats = createStatsForNewPrimary(shardId); RemoteStoreStats stats = new RemoteStoreStats(primaryShardStats, createShardRouting(shardId, true)); try (BytesStreamOutput out = new BytesStreamOutput()) { stats.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); + RemoteSegmentTransferTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); assertEquals(stats.getStats().refreshTimeLagMs, deserializedStats.refreshTimeLagMs); assertEquals(stats.getStats().localRefreshNumber, deserializedStats.localRefreshNumber); assertEquals(stats.getStats().remoteRefreshNumber, deserializedStats.remoteRefreshNumber); @@ -116,12 +116,12 @@ public void testSerializationForPrimaryShard() throws Exception { } public void testSerializationForReplicaShard() throws Exception { - RemoteRefreshSegmentTracker.Stats replicaShardStats = createStatsForNewReplica(shardId); + RemoteSegmentTransferTracker.Stats replicaShardStats = createStatsForNewReplica(shardId); RemoteStoreStats stats = new RemoteStoreStats(replicaShardStats, createShardRouting(shardId, false)); try (BytesStreamOutput out = new BytesStreamOutput()) { stats.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); + RemoteSegmentTransferTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); assertEquals(0, deserializedStats.refreshTimeLagMs); assertEquals(0, deserializedStats.localRefreshNumber); assertEquals(0, deserializedStats.remoteRefreshNumber); @@ -150,12 +150,12 @@ public void testSerializationForReplicaShard() throws Exception { } public void testSerializationForRemoteStoreRestoredPrimaryShard() throws Exception { - RemoteRefreshSegmentTracker.Stats primaryShardStats = createStatsForRemoteStoreRestoredPrimary(shardId); + RemoteSegmentTransferTracker.Stats primaryShardStats = createStatsForRemoteStoreRestoredPrimary(shardId); RemoteStoreStats stats = new RemoteStoreStats(primaryShardStats, createShardRouting(shardId, true)); try (BytesStreamOutput out = new BytesStreamOutput()) { stats.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); + RemoteSegmentTransferTracker.Stats deserializedStats = new RemoteStoreStats(in).getStats(); assertEquals(stats.getStats().refreshTimeLagMs, deserializedStats.refreshTimeLagMs); assertEquals(stats.getStats().localRefreshNumber, deserializedStats.localRefreshNumber); assertEquals(stats.getStats().remoteRefreshNumber, deserializedStats.remoteRefreshNumber); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index 2a554e7abcd44..b3d3faab71aba 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -29,7 +29,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.indices.IndicesService; import org.opensearch.test.FeatureFlagSetter; @@ -86,7 +86,7 @@ public void setUp() throws Exception { Collections.emptySet() ); - when(pressureService.getRemoteRefreshSegmentTracker(any())).thenReturn(mock(RemoteRefreshSegmentTracker.class)); + when(pressureService.getRemoteRefreshSegmentTracker(any())).thenReturn(mock(RemoteSegmentTransferTracker.class)); when(indicesService.indexService(INDEX)).thenReturn(indexService); when(indexService.getIndexSettings()).thenReturn(new IndexSettings(remoteStoreIndexMetadata, Settings.EMPTY)); statsAction = new TransportRemoteStoreStatsAction( diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java index 5ccacd4048596..30c1cb1f4643e 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java @@ -99,7 +99,7 @@ public void testValidateSegmentUploadLag() { pressureService = new RemoteRefreshSegmentPressureService(clusterService, Settings.EMPTY); pressureService.afterIndexShardCreated(indexShard); - RemoteRefreshSegmentTracker pressureTracker = pressureService.getRemoteRefreshSegmentTracker(shardId); + RemoteSegmentTransferTracker pressureTracker = pressureService.getRemoteRefreshSegmentTracker(shardId); pressureTracker.updateLocalRefreshSeqNo(6); // 1. time lag more than dynamic threshold diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java similarity index 92% rename from server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java rename to server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java index 2601521810c0c..82eefa95443cd 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java @@ -24,7 +24,7 @@ import static org.mockito.Mockito.mock; -public class RemoteRefreshSegmentTrackerTests extends OpenSearchTestCase { +public class RemoteSegmentTransferTrackerTests extends OpenSearchTestCase { private RemoteRefreshSegmentPressureSettings pressureSettings; @@ -34,7 +34,7 @@ public class RemoteRefreshSegmentTrackerTests extends OpenSearchTestCase { private ShardId shardId; - private RemoteRefreshSegmentTracker pressureTracker; + private RemoteSegmentTransferTracker pressureTracker; @Override public void setUp() throws Exception { @@ -60,7 +60,7 @@ public void tearDown() throws Exception { } public void testGetShardId() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -70,7 +70,7 @@ public void testGetShardId() { } public void testUpdateLocalRefreshSeqNo() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -82,7 +82,7 @@ public void testUpdateLocalRefreshSeqNo() { } public void testUpdateRemoteRefreshSeqNo() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -94,7 +94,7 @@ public void testUpdateRemoteRefreshSeqNo() { } public void testUpdateLocalRefreshTimeMs() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -106,7 +106,7 @@ public void testUpdateLocalRefreshTimeMs() { } public void testUpdateRemoteRefreshTimeMs() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -118,7 +118,7 @@ public void testUpdateRemoteRefreshTimeMs() { } public void testLastDownloadTimestampMs() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -130,7 +130,7 @@ public void testLastDownloadTimestampMs() { } public void testComputeSeqNoLagOnUpdate() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -145,7 +145,7 @@ public void testComputeSeqNoLagOnUpdate() { } public void testComputeTimeLagOnUpdate() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -162,7 +162,7 @@ public void testComputeTimeLagOnUpdate() { } public void testAddUploadBytesStarted() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -177,7 +177,7 @@ public void testAddUploadBytesStarted() { } public void testAddUploadBytesFailed() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -192,7 +192,7 @@ public void testAddUploadBytesFailed() { } public void testAddUploadBytesSucceeded() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -207,7 +207,7 @@ public void testAddUploadBytesSucceeded() { } public void testAddDownloadBytesStarted() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -222,7 +222,7 @@ public void testAddDownloadBytesStarted() { } public void testAddDownloadBytesFailed() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -237,7 +237,7 @@ public void testAddDownloadBytesFailed() { } public void testAddDownloadBytesSucceeded() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -252,7 +252,7 @@ public void testAddDownloadBytesSucceeded() { } public void testGetInflightUploadBytes() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -268,7 +268,7 @@ public void testGetInflightUploadBytes() { } public void testGetInFlightDownloadBytes() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -284,7 +284,7 @@ public void testGetInFlightDownloadBytes() { } public void testIncrementTotalDownloadsStarted() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -297,7 +297,7 @@ public void testIncrementTotalDownloadsStarted() { } public void testIncrementTotalDownloadsFailed() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -310,7 +310,7 @@ public void testIncrementTotalDownloadsFailed() { } public void testIncrementTotalDownloadsSucceeded() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -323,7 +323,7 @@ public void testIncrementTotalDownloadsSucceeded() { } public void testIncrementTotalUploadsStarted() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -336,7 +336,7 @@ public void testIncrementTotalUploadsStarted() { } public void testIncrementTotalUploadsFailed() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -349,7 +349,7 @@ public void testIncrementTotalUploadsFailed() { } public void testIncrementTotalUploadSucceeded() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -362,7 +362,7 @@ public void testIncrementTotalUploadSucceeded() { } public void testGetInflightUploads() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -379,7 +379,7 @@ public void testGetInflightUploads() { } public void testGetInflightDownloads() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -396,7 +396,7 @@ public void testGetInflightDownloads() { } public void testIncrementRejectionCount() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -409,7 +409,7 @@ public void testIncrementRejectionCount() { } public void testGetConsecutiveFailureCount() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -424,7 +424,7 @@ public void testGetConsecutiveFailureCount() { } public void testComputeBytesLag() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -453,7 +453,7 @@ public void testComputeBytesLag() { } public void testIsUploadBytesAverageReady() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -480,7 +480,7 @@ public void testIsUploadBytesAverageReady() { } public void testIsUploadBytesPerSecAverageReady() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -507,7 +507,7 @@ public void testIsUploadBytesPerSecAverageReady() { } public void testIsUploadTimeMsAverageReady() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -534,7 +534,7 @@ public void testIsUploadTimeMsAverageReady() { } public void testIsDownloadBytesAverageReady() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -561,7 +561,7 @@ public void testIsDownloadBytesAverageReady() { } public void testIsDownloadBytesPerSecAverageReady() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -588,7 +588,7 @@ public void testIsDownloadBytesPerSecAverageReady() { } public void testIsDownloadTimeMsAverageReady() { - pressureTracker = new RemoteRefreshSegmentTracker( + pressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), @@ -615,11 +615,11 @@ public void testIsDownloadTimeMsAverageReady() { } /** - * Tests whether RemoteRefreshSegmentTracker.Stats object generated correctly from RemoteRefreshSegmentTracker. + * Tests whether RemoteSegmentTransferTracker.Stats object generated correctly from RemoteSegmentTransferTracker. * */ public void testStatsObjectCreation() { pressureTracker = constructTracker(); - RemoteRefreshSegmentTracker.Stats pressureTrackerStats = pressureTracker.stats(); + RemoteSegmentTransferTracker.Stats pressureTrackerStats = pressureTracker.stats(); assertEquals(pressureTracker.getShardId(), pressureTrackerStats.shardId); assertEquals(pressureTracker.getTimeMsLag(), (int) pressureTrackerStats.refreshTimeLagMs); assertEquals(pressureTracker.getLocalRefreshSeqNo(), (int) pressureTrackerStats.localRefreshNumber); @@ -645,16 +645,16 @@ public void testStatsObjectCreation() { } /** - * Tests whether RemoteRefreshSegmentTracker.Stats object serialize and deserialize is working fine. + * Tests whether RemoteSegmentTransferTracker.Stats object serialize and deserialize is working fine. * This comes into play during internode data transfer. * */ public void testStatsObjectCreationViaStream() throws IOException { pressureTracker = constructTracker(); - RemoteRefreshSegmentTracker.Stats pressureTrackerStats = pressureTracker.stats(); + RemoteSegmentTransferTracker.Stats pressureTrackerStats = pressureTracker.stats(); try (BytesStreamOutput out = new BytesStreamOutput()) { pressureTrackerStats.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - RemoteRefreshSegmentTracker.Stats deserializedStats = new RemoteRefreshSegmentTracker.Stats(in); + RemoteSegmentTransferTracker.Stats deserializedStats = new RemoteSegmentTransferTracker.Stats(in); assertEquals(deserializedStats.shardId, pressureTrackerStats.shardId); assertEquals((int) deserializedStats.refreshTimeLagMs, (int) pressureTrackerStats.refreshTimeLagMs); assertEquals((int) deserializedStats.localRefreshNumber, (int) pressureTrackerStats.localRefreshNumber); @@ -685,8 +685,8 @@ public void testStatsObjectCreationViaStream() throws IOException { } } - private RemoteRefreshSegmentTracker constructTracker() { - RemoteRefreshSegmentTracker segmentPressureTracker = new RemoteRefreshSegmentTracker( + private RemoteSegmentTransferTracker constructTracker() { + RemoteSegmentTransferTracker segmentPressureTracker = new RemoteSegmentTransferTracker( shardId, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 6f38f080e5035..376971947f857 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -26,7 +26,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; -import org.opensearch.index.remote.RemoteRefreshSegmentTracker; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.Store; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; @@ -253,7 +253,7 @@ public void testRefreshSuccessOnFirstAttempt() throws Exception { assertBusy(() -> assertEquals(0, refreshCountLatch.getCount())); assertBusy(() -> assertEquals(0, successLatch.getCount())); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); - RemoteRefreshSegmentTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); + RemoteSegmentTransferTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); assertNoLagAndTotalUploadsFailed(segmentTracker, 0); } @@ -274,7 +274,7 @@ public void testRefreshSuccessOnSecondAttempt() throws Exception { assertBusy(() -> assertEquals(0, refreshCountLatch.getCount())); assertBusy(() -> assertEquals(0, successLatch.getCount())); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); - RemoteRefreshSegmentTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); + RemoteSegmentTransferTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); assertNoLagAndTotalUploadsFailed(segmentTracker, 1); } @@ -320,11 +320,11 @@ public void testRefreshSuccessOnThirdAttempt() throws Exception { assertBusy(() -> assertEquals(0, refreshCountLatch.getCount())); assertBusy(() -> assertEquals(0, successLatch.getCount())); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); - RemoteRefreshSegmentTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); + RemoteSegmentTransferTracker segmentTracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); assertNoLagAndTotalUploadsFailed(segmentTracker, 2); } - private void assertNoLagAndTotalUploadsFailed(RemoteRefreshSegmentTracker segmentTracker, long totalUploadsFailed) throws Exception { + private void assertNoLagAndTotalUploadsFailed(RemoteSegmentTransferTracker segmentTracker, long totalUploadsFailed) throws Exception { assertBusy(() -> { assertEquals(0, segmentTracker.getBytesLag()); assertEquals(0, segmentTracker.getRefreshSeqNoLag()); @@ -337,7 +337,7 @@ public void testTrackerData() throws Exception { Tuple tuple = mockIndexShardWithRetryAndScheduleRefresh(1); RemoteStoreRefreshListener listener = tuple.v1(); RemoteRefreshSegmentPressureService pressureService = tuple.v2(); - RemoteRefreshSegmentTracker tracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); + RemoteSegmentTransferTracker tracker = pressureService.getRemoteRefreshSegmentTracker(indexShard.shardId()); assertNoLag(tracker); indexDocs(100, randomIntBetween(100, 200)); indexShard.refresh("test"); @@ -345,7 +345,7 @@ public void testTrackerData() throws Exception { assertBusy(() -> assertNoLag(tracker)); } - private void assertNoLag(RemoteRefreshSegmentTracker tracker) { + private void assertNoLag(RemoteSegmentTransferTracker tracker) { assertEquals(0, tracker.getRefreshSeqNoLag()); assertEquals(0, tracker.getBytesLag()); assertEquals(0, tracker.getTimeMsLag()); From d380d1da8484d3383ce59bc8b5bb5b3cd799ae89 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:28:52 +0530 Subject: [PATCH 29/36] Adding JavaDoc for new interface Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../org/opensearch/common/util/SegmentDownloadListener.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java b/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java index 49af6a34891fe..a1bc86ffa934b 100644 --- a/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java +++ b/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java @@ -8,6 +8,11 @@ package org.opensearch.common.util; +/** + * Interface for populating download stats of segments from remote store + * + * @opensearch.internal + */ public interface SegmentDownloadListener { void beforeSync(); From 4a866aea745ecc25631a801a87c0d863ea879642 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:58:13 +0530 Subject: [PATCH 30/36] Manually invoking refresh on ITs Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../java/org/opensearch/remotestore/RemoteStoreStatsIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index ee969849a1b83..adaa5b8caedae 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -213,6 +213,9 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce ); ensureGreen(INDEX_NAME); + // Manually invoke a refresh + refresh(INDEX_NAME); + // Get zero state values // Extract and assert zero state primary stats RemoteStoreStatsResponse zeroStateResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); @@ -525,6 +528,7 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce // Create an index with one primary and one replica shard createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 1)); ensureGreen(INDEX_NAME); + refresh(INDEX_NAME); // Ensure that the index has 0 documents in it assertEquals(0, client().admin().indices().prepareStats(INDEX_NAME).get().getTotal().docs.getCount()); From 91378d19ae8e14769fa7c98e7940403432f30685 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:27:14 +0530 Subject: [PATCH 31/36] Retrigger tests Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> From e843adc4f2fc17fafe075bd698b12a83b0e21d55 Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:56:02 +0530 Subject: [PATCH 32/36] Abstracting out stats population logic to different class Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 67 ++------- .../remotestore/stats/RemoteStoreStats.java | 10 +- .../common/util/SegmentDownloadListener.java | 26 ---- .../remote/RemoteSegmentTransferTracker.java | 87 +----------- .../opensearch/index/shard/IndexShard.java | 125 ++--------------- .../index/shard/SegmentDownloadListener.java | 31 +++++ .../shard/StatsAwareCopyFromRemoteStore.java | 130 ++++++++++++++++++ .../stats/RemoteStoreStatsTestHelper.java | 44 +----- .../stats/RemoteStoreStatsTests.java | 11 -- .../RemoteSegmentTransferTrackerTests.java | 98 +------------ 10 files changed, 193 insertions(+), 436 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java create mode 100644 server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java create mode 100644 server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index adaa5b8caedae..4e11ccc1cd76f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -72,7 +72,7 @@ public void testStatsResponseFromAllNodes() { assertEquals(1, matches.size()); RemoteSegmentTransferTracker.Stats stats = matches.get(0).getStats(); validateUploadStats(stats); - assertEquals(0, stats.totalDownloadsStarted); + assertEquals(0, stats.downloadBytesStarted); } // Step 3 - Enable replicas on the existing indices and ensure that download @@ -93,7 +93,7 @@ public void testStatsResponseFromAllNodes() { RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.totalDownloadsStarted); + assertEquals(0, stats.downloadBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -124,7 +124,7 @@ public void testStatsResponseAllShards() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 3); RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); - assertEquals(0, stats.totalDownloadsStarted); + assertEquals(0, stats.downloadBytesStarted); // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well @@ -138,7 +138,7 @@ public void testStatsResponseAllShards() { stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.totalDownloadsStarted); + assertEquals(0, stats.downloadBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -171,7 +171,7 @@ public void testStatsResponseFromLocalNode() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 1); RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); - assertEquals(0, stats.totalDownloadsStarted); + assertEquals(0, stats.downloadBytesStarted); } changeReplicaCountAndEnsureGreen(1); for (String node : nodes) { @@ -188,7 +188,7 @@ public void testStatsResponseFromLocalNode() { RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.totalDownloadsStarted); + assertEquals(0, stats.downloadBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -240,11 +240,6 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce .collect(Collectors.toList()) .get(0) .getStats(); - assertTrue( - zeroStateReplicaStats.totalDownloadsStarted == 0 - && zeroStateReplicaStats.totalDownloadsSucceeded == 0 - && zeroStateReplicaStats.totalDownloadsFailed == 0 - ); assertTrue(zeroStateReplicaStats.downloadBytesStarted == 0 && zeroStateReplicaStats.downloadBytesSucceeded == 0); // Index documents @@ -269,12 +264,8 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce RemoteSegmentTransferTracker.Stats primaryStats = primaryStatsList.get(0).getStats(); RemoteSegmentTransferTracker.Stats replicaStats = replicaStatsList.get(0).getStats(); // Assert Upload syncs - zero state uploads == download syncs - assertTrue( - primaryStats.totalUploadsStarted > 0 && primaryStats.totalUploadsStarted - 1 == replicaStats.totalDownloadsStarted - ); - assertTrue( - primaryStats.totalUploadsSucceeded > 0 && primaryStats.totalUploadsSucceeded - 1 == replicaStats.totalDownloadsSucceeded - ); + assertTrue(primaryStats.totalUploadsStarted > 0); + assertTrue(primaryStats.totalUploadsSucceeded > 0); assertTrue( replicaStats.downloadBytesStarted > 0 && primaryStats.uploadBytesStarted - zeroStatePrimaryStats.uploadBytesStarted == replicaStats.downloadBytesStarted @@ -285,13 +276,8 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce - zeroStatePrimaryStats.uploadBytesSucceeded == replicaStats.downloadBytesSucceeded ); // Assert zero failures - assertEquals(0, primaryStats.totalDownloadsFailed); assertEquals(0, primaryStats.uploadBytesFailed); - assertEquals(0, replicaStats.totalDownloadsFailed); assertEquals(0, replicaStats.downloadBytesFailed); - - // Assert last segment size uploaded = last segment size downloaded - assertEquals(replicaStats.lastSuccessfulSegmentDownloadBytes, primaryStats.lastSuccessfulRemoteRefreshBytes); }, 60, TimeUnit.SECONDS); } } @@ -342,11 +328,6 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()); zeroStateReplicaStats.forEach(stats -> { - assertTrue( - stats.getStats().totalDownloadsStarted == 0 - && stats.getStats().totalDownloadsSucceeded == 0 - && stats.getStats().totalDownloadsFailed == 0 - ); assertTrue(stats.getStats().downloadBytesStarted == 0 && stats.getStats().downloadBytesSucceeded == 0); }); @@ -361,45 +342,29 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr assertEquals(currentNodesInCluster, response.getSuccessfulShards()); long uploadsStarted = 0, uploadsSucceeded = 0, uploadsFailed = 0; long uploadBytesStarted = 0, uploadBytesSucceeded = 0, uploadBytesFailed = 0; - List downloadsStarted = new ArrayList<>(), downloadsSucceeded = new ArrayList<>(), downloadsFailed = - new ArrayList<>(); List downloadBytesStarted = new ArrayList<>(), downloadBytesSucceeded = new ArrayList<>(), downloadBytesFailed = new ArrayList<>(); - long lastUploadedSegmentSize = 0; - List lastDownloadedSegmentSize = new ArrayList<>(); // Assert that stats for primary shard and replica shard set are equal for (RemoteStoreStats eachStatsObject : response.getRemoteStoreStats()) { RemoteSegmentTransferTracker.Stats stats = eachStatsObject.getStats(); if (eachStatsObject.getShardRouting().primary()) { - uploadsStarted = stats.totalUploadsStarted; - uploadsSucceeded = stats.totalUploadsSucceeded; - uploadsFailed = stats.totalUploadsFailed; uploadBytesStarted = stats.uploadBytesStarted; uploadBytesSucceeded = stats.uploadBytesSucceeded; uploadBytesFailed = stats.uploadBytesFailed; - lastUploadedSegmentSize = stats.lastSuccessfulRemoteRefreshBytes; } else { - downloadsStarted.add(stats.totalDownloadsStarted); - downloadsSucceeded.add(stats.totalDownloadsSucceeded); - downloadsFailed.add(stats.totalDownloadsFailed); downloadBytesStarted.add(stats.downloadBytesStarted); downloadBytesSucceeded.add(stats.downloadBytesSucceeded); downloadBytesFailed.add(stats.downloadBytesFailed); - lastDownloadedSegmentSize.add(stats.lastSuccessfulSegmentDownloadBytes); } } assertEquals(0, uploadsFailed); assertEquals(0, uploadBytesFailed); for (int j = 0; j < response.getSuccessfulShards() - 1; j++) { - assertEquals(uploadsStarted - 1, (long) downloadsStarted.get(j)); - assertEquals(uploadsSucceeded - 1, (long) downloadsSucceeded.get(j)); - assertEquals(0, (long) downloadsFailed.get(j)); assertEquals(uploadBytesStarted - zeroStatePrimaryStats.uploadBytesStarted, (long) downloadBytesStarted.get(j)); assertEquals(uploadBytesSucceeded - zeroStatePrimaryStats.uploadBytesSucceeded, (long) downloadBytesSucceeded.get(j)); assertEquals(0, (long) downloadBytesFailed.get(j)); - assertEquals(lastUploadedSegmentSize, (long) lastDownloadedSegmentSize.get(j)); } }, 60, TimeUnit.SECONDS); } @@ -516,11 +481,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { assertTrue( segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0 ); - assertTrue( - segmentTracker.totalDownloadsStarted > 0 - && segmentTracker.totalDownloadsSucceeded > 0 - && segmentTracker.totalDownloadsFailed == 0 - ); + assertTrue(segmentTracker.downloadBytesStarted > 0 && segmentTracker.downloadBytesSucceeded > 0); }); } @@ -549,11 +510,7 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce && segmentTracker.totalUploadsFailed == 0 ); } else { - assertTrue( - segmentTracker.totalDownloadsStarted == 0 - && segmentTracker.totalDownloadsSucceeded == 0 - && segmentTracker.totalDownloadsFailed == 0 - ); + assertTrue(segmentTracker.downloadBytesStarted == 0 && segmentTracker.downloadBytesSucceeded == 0); } }); }, 5, TimeUnit.SECONDS); @@ -608,16 +565,12 @@ private void validateUploadStats(RemoteSegmentTransferTracker.Stats stats) { private void validateDownloadStats(RemoteSegmentTransferTracker.Stats stats) { assertTrue(stats.lastDownloadTimestampMs > 0); - assertTrue(stats.totalDownloadsStarted > 0); - assertTrue(stats.totalDownloadsSucceeded > 0); - assertEquals(stats.totalDownloadsFailed, 0); assertTrue(stats.downloadBytesStarted > 0); assertTrue(stats.downloadBytesSucceeded > 0); assertEquals(stats.downloadBytesFailed, 0); assertTrue(stats.lastSuccessfulSegmentDownloadBytes > 0); assertTrue(stats.downloadBytesMovingAverage > 0); assertTrue(stats.downloadBytesPerSecMovingAverage > 0); - assertTrue(stats.downloadTimeMovingAverage > 0); } // Validate if the shardRouting obtained from cluster state contains the exact same routing object diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index ca92e8fd082df..b4541dc66ced7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -54,7 +54,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.SEGMENT); builder.startObject(SubFields.DOWNLOAD); // Ensuring that we are not showing 0 metrics to the user - if (remoteSegmentShardStats.totalDownloadsStarted != 0) { + if (remoteSegmentShardStats.downloadBytesStarted != 0) { buildDownloadStats(builder); } builder.endObject(); @@ -106,11 +106,6 @@ private void buildUploadStats(XContentBuilder builder) throws IOException { private void buildDownloadStats(XContentBuilder builder) throws IOException { builder.field(DownloadStatsFields.LAST_SYNC_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); - builder.startObject(DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE) - .field(SubFields.STARTED, remoteSegmentShardStats.totalDownloadsStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.totalDownloadsSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.totalDownloadsFailed); - builder.endObject(); builder.startObject(DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES) .field(SubFields.STARTED, remoteSegmentShardStats.downloadBytesStarted) .field(SubFields.SUCCEEDED, remoteSegmentShardStats.downloadBytesSucceeded) @@ -123,9 +118,6 @@ private void buildDownloadStats(XContentBuilder builder) throws IOException { builder.startObject(DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC) .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesPerSecMovingAverage); builder.endObject(); - builder.startObject(DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadTimeMovingAverage); - builder.endObject(); } private void buildShardRouting(XContentBuilder builder) throws IOException { diff --git a/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java b/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java deleted file mode 100644 index a1bc86ffa934b..0000000000000 --- a/server/src/main/java/org/opensearch/common/util/SegmentDownloadListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.common.util; - -/** - * Interface for populating download stats of segments from remote store - * - * @opensearch.internal - */ -public interface SegmentDownloadListener { - void beforeSync(); - - void afterSync(long downloadedFilesSize, long startTimeInMs); - - void beforeFileDownload(long fileSize); - - void afterFileDownload(long fileSize); - - void fileDownloadFailed(long fileSize); -} diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java index 9f68509049a79..6f1f48a01c776 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java @@ -134,21 +134,6 @@ public class RemoteSegmentTransferTracker { */ private volatile long totalUploadsSucceeded; - /** - * Cumulative sum of count of segment file downloads that have started. - */ - private volatile long totalDownloadsStarted; - - /** - * Cumulative sum of count of segment file downloads that have succeeded. - */ - private volatile long totalDownloadsSucceeded; - - /** - * Cumulative sum of count of segment file downloads that have failed. - */ - private volatile long totalDownloadsFailed; - /** * Cumulative sum of rejection counts for this shard. */ @@ -218,11 +203,6 @@ public class RemoteSegmentTransferTracker { private final Object uploadTimeMsMutex = new Object(); - /** - * Provides moving average over the last N overall download time (in millis) of segments downloaded from the remote store. - */ - private volatile MovingAverage downloadTimeMovingAverageReference; - private final int SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE = 20; public RemoteSegmentTransferTracker( @@ -244,7 +224,6 @@ public RemoteSegmentTransferTracker( uploadTimeMsMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadTimeMsMovingAverageWindowSize)); downloadBytesMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); downloadBytesPerSecMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); - downloadTimeMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); latestLocalFileNameLengthMap = new HashMap<>(); } @@ -438,34 +417,6 @@ public long getInflightUploads() { return totalUploadsStarted - totalUploadsFailed - totalUploadsSucceeded; } - public long getTotalDownloadsStarted() { - return totalDownloadsStarted; - } - - public void incrementTotalDownloadsStarted() { - totalDownloadsStarted += 1; - } - - public long getTotalDownloadsFailed() { - return totalDownloadsFailed; - } - - public void incrementTotalDownloadsFailed() { - totalDownloadsFailed += 1; - } - - public long getTotalDownloadsSucceeded() { - return totalDownloadsSucceeded; - } - - public void incrementTotalDownloadsSucceeded() { - totalDownloadsSucceeded += 1; - } - - public long getInflightDownloads() { - return totalDownloadsStarted - totalDownloadsFailed - totalDownloadsSucceeded; - } - public long getRejectionCount() { return rejectionCount.get(); } @@ -552,7 +503,7 @@ boolean isDownloadBytesAverageReady() { return downloadBytesMovingAverageReference.getAverage(); } - public void addDownloadBytes(long size) { + public void updateLastDownloadedSegmentSize(long size) { lastSuccessfulSegmentDownloadBytes = size; this.downloadBytesMovingAverageReference.record(size); } @@ -619,18 +570,6 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { } } - boolean isDownloadTimeAverageReady() { - return downloadTimeMovingAverageReference.isReady(); - } - - double getDownloadTimeAverage() { - return downloadTimeMovingAverageReference.getAverage(); - } - - public void addDownloadTime(long timeMs) { - this.downloadTimeMovingAverageReference.record(timeMs); - } - public RemoteSegmentTransferTracker.Stats stats() { return new RemoteSegmentTransferTracker.Stats( shardId, @@ -649,9 +588,6 @@ public RemoteSegmentTransferTracker.Stats stats() { totalUploadsStarted, totalUploadsSucceeded, totalUploadsFailed, - totalDownloadsStarted, - totalDownloadsSucceeded, - totalDownloadsFailed, rejectionCount.get(), failures.length(), lastSuccessfulRemoteRefreshBytes, @@ -661,7 +597,6 @@ public RemoteSegmentTransferTracker.Stats stats() { lastSuccessfulSegmentDownloadBytes, downloadBytesMovingAverageReference.getAverage(), downloadBytesPerSecMovingAverageReference.getAverage(), - downloadTimeMovingAverageReference.getAverage(), getBytesLag() ); } @@ -689,9 +624,6 @@ public static class Stats implements Writeable { public final long totalUploadsStarted; public final long totalUploadsFailed; public final long totalUploadsSucceeded; - public final long totalDownloadsStarted; - public final long totalDownloadsFailed; - public final long totalDownloadsSucceeded; public final long rejectionCount; public final long consecutiveFailuresCount; public final long lastSuccessfulRemoteRefreshBytes; @@ -701,7 +633,6 @@ public static class Stats implements Writeable { public final long lastSuccessfulSegmentDownloadBytes; public final double downloadBytesMovingAverage; public final double downloadBytesPerSecMovingAverage; - public final double downloadTimeMovingAverage; public final long bytesLag; public Stats( @@ -721,9 +652,6 @@ public Stats( long totalUploadsStarted, long totalUploadsSucceeded, long totalUploadsFailed, - long totalDownloadsStarted, - long totalDownloadsSucceeded, - long totalDownloadsFailed, long rejectionCount, long consecutiveFailuresCount, long lastSuccessfulRemoteRefreshBytes, @@ -733,7 +661,6 @@ public Stats( long lastSuccessfulSegmentDownloadBytes, double downloadBytesMovingAverage, double downloadBytesPerSecMovingAverage, - double downloadTimeMovingAverage, long bytesLag ) { this.shardId = shardId; @@ -752,9 +679,6 @@ public Stats( this.totalUploadsStarted = totalUploadsStarted; this.totalUploadsFailed = totalUploadsFailed; this.totalUploadsSucceeded = totalUploadsSucceeded; - this.totalDownloadsStarted = totalDownloadsStarted; - this.totalDownloadsFailed = totalDownloadsFailed; - this.totalDownloadsSucceeded = totalDownloadsSucceeded; this.rejectionCount = rejectionCount; this.consecutiveFailuresCount = consecutiveFailuresCount; this.lastSuccessfulRemoteRefreshBytes = lastSuccessfulRemoteRefreshBytes; @@ -764,7 +688,6 @@ public Stats( this.lastSuccessfulSegmentDownloadBytes = lastSuccessfulSegmentDownloadBytes; this.downloadBytesMovingAverage = downloadBytesMovingAverage; this.downloadBytesPerSecMovingAverage = downloadBytesPerSecMovingAverage; - this.downloadTimeMovingAverage = downloadTimeMovingAverage; this.bytesLag = bytesLag; } @@ -793,13 +716,9 @@ public Stats(StreamInput in) throws IOException { this.downloadBytesStarted = in.readLong(); this.downloadBytesFailed = in.readLong(); this.downloadBytesSucceeded = in.readLong(); - this.totalDownloadsStarted = in.readLong(); - this.totalDownloadsFailed = in.readLong(); - this.totalDownloadsSucceeded = in.readLong(); this.lastSuccessfulSegmentDownloadBytes = in.readLong(); this.downloadBytesMovingAverage = in.readDouble(); this.downloadBytesPerSecMovingAverage = in.readDouble(); - this.downloadTimeMovingAverage = in.readDouble(); } catch (IOException e) { throw e; } @@ -830,13 +749,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(downloadBytesStarted); out.writeLong(downloadBytesFailed); out.writeLong(downloadBytesSucceeded); - out.writeLong(totalDownloadsStarted); - out.writeLong(totalDownloadsFailed); - out.writeLong(totalDownloadsSucceeded); out.writeLong(lastSuccessfulSegmentDownloadBytes); out.writeDouble(downloadBytesMovingAverage); out.writeDouble(downloadBytesPerSecMovingAverage); - out.writeDouble(downloadTimeMovingAverage); } } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 2d2b306c6f708..e79c2cb143a38 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -58,7 +58,6 @@ import org.apache.lucene.util.ThreadInterruptedException; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.cluster.metadata.DataStream; -import org.opensearch.common.util.SegmentDownloadListener; import org.opensearch.core.Assertions; import org.opensearch.ExceptionsHelper; import org.opensearch.OpenSearchException; @@ -147,7 +146,6 @@ import org.opensearch.index.recovery.RecoveryStats; import org.opensearch.index.refresh.RefreshStats; import org.opensearch.index.remote.RemoteRefreshSegmentPressureService; -import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.index.search.stats.SearchStats; import org.opensearch.index.search.stats.ShardSearchStats; import org.opensearch.index.seqno.ReplicationTracker; @@ -4765,7 +4763,6 @@ public void syncSegmentsFromGivenRemoteSegmentStore( } } - // TODO: Move this method to a generic util class along-with the segment upload code private String copySegmentFiles( Directory storeDirectory, RemoteSegmentStoreDirectory sourceRemoteDirectory, @@ -4773,14 +4770,12 @@ private String copySegmentFiles( Map uploadedSegments, boolean overrideLocal ) throws IOException { - boolean isRemoteStoreEnabled = indexSettings.isRemoteStoreEnabled(); - SegmentDownloadListener segmentDownloadListener = initializeSegmentDownloadListener( - remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId) - ); List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); - List segmentsToDownload = new ArrayList<>(); - long sizeOfDownloadedSegments = 0; + StatsAwareCopyFromRemoteStore statsAwareSegmentCopy = new StatsAwareCopyFromRemoteStore( + remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), + indexSettings.isRemoteStoreEnabled() + ); String segmentNFile = null; try { Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); @@ -4789,41 +4784,21 @@ private String copySegmentFiles( storeDirectory.deleteFile(file); } } - // Compute the number of files to download - // Required so that we don't publish metrics if there are no files to copy over for (String file : uploadedSegments.keySet()) { long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - segmentsToDownload.add(file); + statsAwareSegmentCopy.performCopy( + storeDirectory, + sourceRemoteDirectory, + file, + file, + uploadedSegments.get(file).getLength(), + IOContext.DEFAULT + ); + downloadedSegments.add(file); } else { skippedSegments.add(file); } - } - // Upload stats before starting Segment downloads - // Adding a check to prevent stats being published for NoOp runs - if (!segmentsToDownload.isEmpty() && isRemoteStoreEnabled) { - segmentDownloadListener.beforeSync(); - } - long startTimeInMs = System.currentTimeMillis(); - // Copying segments files to local store directory from remote store directory (Shard Recovery) - for (String file : segmentsToDownload) { - sizeOfDownloadedSegments = statsAwareCopyFromRemoteStore( - storeDirectory, - sourceRemoteDirectory, - uploadedSegments, - segmentDownloadListener, - isRemoteStoreEnabled, - downloadedSegments, - sizeOfDownloadedSegments, - file - ); - } - // Update stats after download batch is completed - if (!segmentsToDownload.isEmpty() && isRemoteStoreEnabled) { - segmentDownloadListener.afterSync(sizeOfDownloadedSegments, startTimeInMs); - } - // Copying segment files over to target remote directory from local store directory (Snapshot) - for (String file : uploadedSegments.keySet()) { if (targetRemoteDirectory != null) { targetRemoteDirectory.copyFrom(storeDirectory, file, file, IOContext.DEFAULT); } @@ -4839,80 +4814,6 @@ private String copySegmentFiles( return segmentNFile; } - private SegmentDownloadListener initializeSegmentDownloadListener(RemoteSegmentTransferTracker downloadStatsTracker) { - return new SegmentDownloadListener() { - @Override - public void beforeSync() { - downloadStatsTracker.incrementTotalDownloadsStarted(); - } - - @Override - public void afterSync(long downloadedFilesSize, long startTimeInMs) { - long currentTimeInMs = System.currentTimeMillis(); - downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); - downloadStatsTracker.incrementTotalDownloadsSucceeded(); - downloadStatsTracker.addDownloadBytes(downloadedFilesSize); - long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); - downloadStatsTracker.addDownloadTime(timeTakenInMS); - downloadStatsTracker.addDownloadBytesPerSec((downloadedFilesSize * 1_000L) / timeTakenInMS); - } - - @Override - public void beforeFileDownload(long fileSize) { - downloadStatsTracker.addDownloadBytesStarted(fileSize); - } - - @Override - public void afterFileDownload(long fileSize) { - downloadStatsTracker.addDownloadBytesSucceeded(fileSize); - } - - @Override - public void fileDownloadFailed(long fileSize) { - downloadStatsTracker.addDownloadBytesFailed(fileSize); - downloadStatsTracker.incrementTotalDownloadsFailed(); - } - }; - } - - private long statsAwareCopyFromRemoteStore( - Directory storeDirectory, - RemoteSegmentStoreDirectory sourceRemoteDirectory, - Map uploadedSegments, - SegmentDownloadListener segmentDownloadListener, - boolean isRemoteStoreEnabled, - List downloadedSegments, - long sizeOfDownloadedSegments, - String file - ) throws IOException { - long fileSize = uploadedSegments.get(file).getLength(); - boolean fileDownloadComplete = false; - // Download bytes stats population: - // - Increments attempted bytes just before invoking copyFrom method - // - Increment succeeded bytes after copyFrom method executes successfully - // - Increment failed bytes when copyFrom method fails - try { - // Adding file size to the amount of bytes attempted for download - if (isRemoteStoreEnabled) { - segmentDownloadListener.beforeFileDownload(fileSize); - } - storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - fileDownloadComplete = true; - downloadedSegments.add(file); - sizeOfDownloadedSegments += fileSize; - // Adding file size to the amount of bytes completed in download - if (isRemoteStoreEnabled) { - segmentDownloadListener.afterFileDownload(fileSize); - } - } finally { - // Increment download failure stats if the `copyFrom` method above throws an exception - if (!fileDownloadComplete && isRemoteStoreEnabled) { - segmentDownloadListener.fileDownloadFailed(fileSize); - } - } - return sizeOfDownloadedSegments; - } - private boolean localDirectoryContains(Directory localDirectory, String file, long checksum) { try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) { if (checksum == CodecUtil.retrieveChecksum(indexInput)) { diff --git a/server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java b/server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java new file mode 100644 index 0000000000000..ac78a4b3e0bec --- /dev/null +++ b/server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +/** + * Interface with hooks executed on segment download events + * + * @opensearch.internal + */ +public interface SegmentDownloadListener { + /** + * Executed before segment download starts + */ + void beforeDownload(); + + /** + * Executed after segment download completes + */ + void afterDownload(); + + /** + * Executed if segment download fails + */ + void downloadFailed(); +} diff --git a/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java b/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java new file mode 100644 index 0000000000000..bbb1fee090005 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java @@ -0,0 +1,130 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.shard; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.opensearch.index.remote.RemoteSegmentTransferTracker; +import org.opensearch.index.store.RemoteSegmentStoreDirectory; + +import java.io.IOException; + +/** + * Copies over files from a remote store and populates remote store stats accordingly + * + * @opensearch.internal + */ +public class StatsAwareCopyFromRemoteStore { + private final RemoteStoreDownloadStatsListener downloadStatsListener; + + public StatsAwareCopyFromRemoteStore(RemoteSegmentTransferTracker downloadStatsTracker, boolean isRemoteStoreEnabled) { + this.downloadStatsListener = new RemoteStoreDownloadStatsListener(downloadStatsTracker, isRemoteStoreEnabled); + } + + /** + * Encapsulates the copyFrom method and populates remote store segment download stats + * @param storeDirectory: Destination directory implementation to copy segments to + * @param remoteDirectory: Source {@link RemoteSegmentStoreDirectory} to copy segments from + * @param srcFile: Source file name + * @param destFile: Source file name + * @param fileSize: Size of the file being downloaded + * @param ioContext: {@link IOContext} object to be used in {@link Directory#copyFrom} method + * @throws IOException if an error occurs in copying files + */ + public void performCopy( + Directory storeDirectory, + RemoteSegmentStoreDirectory remoteDirectory, + String srcFile, + String destFile, + long fileSize, + IOContext ioContext + ) throws IOException { + downloadStatsListener.updateFileSize(fileSize); + downloadStatsListener.beforeDownload(); + boolean copySuccessful = false; + try { + storeDirectory.copyFrom(remoteDirectory, srcFile, destFile, ioContext); + copySuccessful = true; + downloadStatsListener.afterDownload(); + } finally { + if (!copySuccessful) { + downloadStatsListener.downloadFailed(); + } + } + } + + /** + * Implementation of {@link SegmentDownloadListener} to populate remote store state related to segment downloads + * + * @opensearch.internal + */ + public static class RemoteStoreDownloadStatsListener implements SegmentDownloadListener { + private final RemoteSegmentTransferTracker downloadStatsTracker; + private final boolean isRemoteStoreEnabled; + private long startTimeInMs; + private long fileSize; + + public RemoteStoreDownloadStatsListener(RemoteSegmentTransferTracker downloadStatsTracker, boolean isRemoteStoreEnabled) { + this.downloadStatsTracker = downloadStatsTracker; + this.isRemoteStoreEnabled = isRemoteStoreEnabled; + } + + /** + * Updates the current segment file's size. + * This would be used to update the tracker values + * @param fileSize: Size of the incoming segment file + */ + public void updateFileSize(long fileSize) { + this.fileSize = fileSize; + } + + /** + * Updates the amount of bytes attempted for download + */ + @Override + public void beforeDownload() { + if (!isRemoteStoreEnabled) { + return; + } + startTimeInMs = System.currentTimeMillis(); + downloadStatsTracker.addDownloadBytesStarted(fileSize); + } + + /** + * Updates + * - The amount of bytes that has been successfully downloaded from the remote store + * - The last successful download completion timestamp + * - The last successfully downloaded segment size + * - The speed of segment download (in bytes/sec) + */ + @Override + public void afterDownload() { + if (!isRemoteStoreEnabled) { + return; + } + downloadStatsTracker.addDownloadBytesSucceeded(fileSize); + downloadStatsTracker.updateLastDownloadedSegmentSize(fileSize); + long currentTimeInMs = System.currentTimeMillis(); + downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); + long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); + downloadStatsTracker.addDownloadBytesPerSec((fileSize * 1_000L) / timeTakenInMS); + } + + /** + * Updates the amount of bytes failed in download + */ + @Override + public void downloadFailed() { + if (!isRemoteStoreEnabled) { + return; + } + downloadStatsTracker.addDownloadBytesFailed(fileSize); + } + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index c400126bb80a3..ce056cf4d36d2 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -43,8 +43,6 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewPrimary(ShardId shard 5, 5, 0, - 0, - 0, 3, 2, 5, @@ -53,9 +51,7 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewPrimary(ShardId shard 4, 0, 0, - 0, - 0, - 5 + 0 ); } @@ -77,9 +73,6 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewReplica(ShardId shard 0, 0, 0, - 10, - 10, - 0, 0, 0, 0, @@ -89,8 +82,7 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewReplica(ShardId shard 10, 10, 10, - 10, - 0 + 10 ); } @@ -112,9 +104,6 @@ static RemoteSegmentTransferTracker.Stats createStatsForRemoteStoreRestoredPrima 10, 10, 0, - 10, - 10, - 0, 0, 0, 100, @@ -124,8 +113,7 @@ static RemoteSegmentTransferTracker.Stats createStatsForRemoteStoreRestoredPrima 10, 10, 10, - 10, - 0 + 10 ); } @@ -155,29 +143,11 @@ static void compareStatsResponse( Map segmentDownloads = ((Map) segment.get(RemoteStoreStats.SubFields.DOWNLOAD)); Map segmentUploads = ((Map) segment.get(RemoteStoreStats.SubFields.UPLOAD)); - if (statsTracker.totalDownloadsStarted != 0) { + if (statsTracker.downloadBytesStarted != 0) { assertEquals( segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_SYNC_TIMESTAMP), (int) statsTracker.lastDownloadTimestampMs ); - assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( - RemoteStoreStats.SubFields.STARTED - ), - (int) statsTracker.totalDownloadsStarted - ); - assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( - RemoteStoreStats.SubFields.SUCCEEDED - ), - (int) statsTracker.totalDownloadsSucceeded - ); - assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_SYNCS_FROM_REMOTE)).get( - RemoteStoreStats.SubFields.FAILED - ), - (int) statsTracker.totalDownloadsFailed - ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.STARTED @@ -214,12 +184,6 @@ static void compareStatsResponse( ), statsTracker.downloadBytesPerSecMovingAverage ); - assertEquals( - ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_LATENCY_IN_MILLIS)).get( - RemoteStoreStats.SubFields.MOVING_AVG - ), - statsTracker.downloadTimeMovingAverage - ); } else { assertTrue(segmentDownloads.isEmpty()); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index b2193fd658660..05bd642a34587 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -103,9 +103,6 @@ public void testSerializationForPrimaryShard() throws Exception { assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); - assertEquals(0, deserializedStats.totalDownloadsStarted); - assertEquals(0, deserializedStats.totalDownloadsSucceeded); - assertEquals(0, deserializedStats.totalDownloadsFailed); assertEquals(0, deserializedStats.downloadBytesStarted); assertEquals(0, deserializedStats.downloadBytesFailed); assertEquals(0, deserializedStats.downloadBytesSucceeded); @@ -134,16 +131,12 @@ public void testSerializationForReplicaShard() throws Exception { assertEquals(0, deserializedStats.rejectionCount); assertEquals(0, deserializedStats.consecutiveFailuresCount); assertEquals(0, deserializedStats.bytesLag); - assertEquals(stats.getStats().totalDownloadsStarted, deserializedStats.totalDownloadsStarted); - assertEquals(stats.getStats().totalDownloadsSucceeded, deserializedStats.totalDownloadsSucceeded); - assertEquals(stats.getStats().totalDownloadsFailed, deserializedStats.totalDownloadsFailed); assertEquals(stats.getStats().downloadBytesStarted, deserializedStats.downloadBytesStarted); assertEquals(stats.getStats().downloadBytesFailed, deserializedStats.downloadBytesFailed); assertEquals(stats.getStats().downloadBytesSucceeded, deserializedStats.downloadBytesSucceeded); assertEquals(stats.getStats().lastSuccessfulSegmentDownloadBytes, deserializedStats.lastSuccessfulSegmentDownloadBytes); assertEquals(stats.getStats().lastDownloadTimestampMs, deserializedStats.lastDownloadTimestampMs); assertEquals(stats.getStats().downloadBytesPerSecMovingAverage, deserializedStats.downloadBytesPerSecMovingAverage, 0); - assertEquals(stats.getStats().downloadTimeMovingAverage, deserializedStats.downloadTimeMovingAverage, 0); assertEquals(stats.getStats().downloadBytesMovingAverage, deserializedStats.downloadBytesMovingAverage, 0); } } @@ -171,16 +164,12 @@ public void testSerializationForRemoteStoreRestoredPrimaryShard() throws Excepti assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); - assertEquals(stats.getStats().totalDownloadsStarted, deserializedStats.totalDownloadsStarted); - assertEquals(stats.getStats().totalDownloadsSucceeded, deserializedStats.totalDownloadsSucceeded); - assertEquals(stats.getStats().totalDownloadsFailed, deserializedStats.totalDownloadsFailed); assertEquals(stats.getStats().downloadBytesStarted, deserializedStats.downloadBytesStarted); assertEquals(stats.getStats().downloadBytesFailed, deserializedStats.downloadBytesFailed); assertEquals(stats.getStats().downloadBytesSucceeded, deserializedStats.downloadBytesSucceeded); assertEquals(stats.getStats().lastSuccessfulSegmentDownloadBytes, deserializedStats.lastSuccessfulSegmentDownloadBytes); assertEquals(stats.getStats().lastDownloadTimestampMs, deserializedStats.lastDownloadTimestampMs); assertEquals(stats.getStats().downloadBytesPerSecMovingAverage, deserializedStats.downloadBytesPerSecMovingAverage, 0); - assertEquals(stats.getStats().downloadTimeMovingAverage, deserializedStats.downloadTimeMovingAverage, 0); assertEquals(stats.getStats().downloadBytesMovingAverage, deserializedStats.downloadBytesMovingAverage, 0); } } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java index 82eefa95443cd..0e4c87d604813 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java @@ -283,45 +283,6 @@ public void testGetInFlightDownloadBytes() { assertEquals(bytesStarted - bytesSucceeded - bytesFailed, pressureTracker.getInflightDownloadBytes()); } - public void testIncrementTotalDownloadsStarted() { - pressureTracker = new RemoteSegmentTransferTracker( - shardId, - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() - ); - pressureTracker.incrementTotalDownloadsStarted(); - assertEquals(1, pressureTracker.getTotalDownloadsStarted()); - pressureTracker.incrementTotalDownloadsStarted(); - assertEquals(2, pressureTracker.getTotalDownloadsStarted()); - } - - public void testIncrementTotalDownloadsFailed() { - pressureTracker = new RemoteSegmentTransferTracker( - shardId, - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() - ); - pressureTracker.incrementTotalDownloadsFailed(); - assertEquals(1, pressureTracker.getTotalDownloadsFailed()); - pressureTracker.incrementTotalDownloadsFailed(); - assertEquals(2, pressureTracker.getTotalDownloadsFailed()); - } - - public void testIncrementTotalDownloadsSucceeded() { - pressureTracker = new RemoteSegmentTransferTracker( - shardId, - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() - ); - pressureTracker.incrementTotalDownloadsSucceeded(); - assertEquals(1, pressureTracker.getTotalDownloadsSucceeded()); - pressureTracker.incrementTotalDownloadsSucceeded(); - assertEquals(2, pressureTracker.getTotalDownloadsSucceeded()); - } - public void testIncrementTotalUploadsStarted() { pressureTracker = new RemoteSegmentTransferTracker( shardId, @@ -378,23 +339,6 @@ public void testGetInflightUploads() { assertEquals(0, pressureTracker.getInflightUploads()); } - public void testGetInflightDownloads() { - pressureTracker = new RemoteSegmentTransferTracker( - shardId, - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() - ); - pressureTracker.incrementTotalDownloadsStarted(); - assertEquals(1, pressureTracker.getInflightDownloads()); - pressureTracker.incrementTotalDownloadsStarted(); - assertEquals(2, pressureTracker.getInflightDownloads()); - pressureTracker.incrementTotalDownloadsSucceeded(); - assertEquals(1, pressureTracker.getInflightDownloads()); - pressureTracker.incrementTotalDownloadsFailed(); - assertEquals(0, pressureTracker.getInflightDownloads()); - } - public void testIncrementRejectionCount() { pressureTracker = new RemoteSegmentTransferTracker( shardId, @@ -544,18 +488,18 @@ public void testIsDownloadBytesAverageReady() { long sum = 0; for (int i = 1; i < 20; i++) { - pressureTracker.addDownloadBytes(i); + pressureTracker.updateLastDownloadedSegmentSize(i); sum += i; assertFalse(pressureTracker.isDownloadBytesAverageReady()); assertEquals((double) sum / i, pressureTracker.getDownloadBytesAverage(), 0.0d); } - pressureTracker.addDownloadBytes(20); + pressureTracker.updateLastDownloadedSegmentSize(20); sum += 20; assertTrue(pressureTracker.isDownloadBytesAverageReady()); assertEquals((double) sum / 20, pressureTracker.getDownloadBytesAverage(), 0.0d); - pressureTracker.addDownloadBytes(100); + pressureTracker.updateLastDownloadedSegmentSize(100); sum = sum + 100 - 1; assertEquals((double) sum / 20, pressureTracker.getDownloadBytesAverage(), 0.0d); } @@ -587,33 +531,6 @@ public void testIsDownloadBytesPerSecAverageReady() { assertEquals((double) sum / 20, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); } - public void testIsDownloadTimeMsAverageReady() { - pressureTracker = new RemoteSegmentTransferTracker( - shardId, - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() - ); - assertFalse(pressureTracker.isDownloadTimeAverageReady()); - - long sum = 0; - for (int i = 1; i < 20; i++) { - pressureTracker.addDownloadTime(i); - sum += i; - assertFalse(pressureTracker.isDownloadTimeAverageReady()); - assertEquals((double) sum / i, pressureTracker.getDownloadTimeAverage(), 0.0d); - } - - pressureTracker.addDownloadTime(20); - sum += 20; - assertTrue(pressureTracker.isDownloadTimeAverageReady()); - assertEquals((double) sum / 20, pressureTracker.getDownloadTimeAverage(), 0.0d); - - pressureTracker.addDownloadTime(100); - sum = sum + 100 - 1; - assertEquals((double) sum / 20, pressureTracker.getDownloadTimeAverage(), 0.0d); - } - /** * Tests whether RemoteSegmentTransferTracker.Stats object generated correctly from RemoteSegmentTransferTracker. * */ @@ -637,11 +554,8 @@ public void testStatsObjectCreation() { assertEquals(pressureTracker.getTotalUploadsSucceeded(), (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals(pressureTracker.getTotalUploadsFailed(), (int) pressureTrackerStats.totalUploadsFailed); assertEquals(pressureTracker.getLastDownloadTimestampMs(), pressureTrackerStats.lastDownloadTimestampMs); - assertEquals(pressureTracker.getTotalDownloadsStarted(), pressureTrackerStats.totalDownloadsStarted); - assertEquals(pressureTracker.getTotalDownloadsSucceeded(), pressureTrackerStats.totalDownloadsSucceeded); assertEquals(pressureTracker.getDownloadBytesStarted(), pressureTrackerStats.downloadBytesStarted); assertEquals(pressureTracker.getDownloadBytesSucceeded(), pressureTrackerStats.downloadBytesStarted); - assertEquals(pressureTracker.getDownloadTimeAverage(), pressureTrackerStats.downloadTimeMovingAverage, 0); } /** @@ -676,11 +590,8 @@ public void testStatsObjectCreationViaStream() throws IOException { assertEquals((int) deserializedStats.totalUploadsSucceeded, (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals((int) deserializedStats.totalUploadsFailed, (int) pressureTrackerStats.totalUploadsFailed); assertEquals(deserializedStats.lastDownloadTimestampMs, pressureTrackerStats.lastDownloadTimestampMs); - assertEquals(deserializedStats.totalDownloadsStarted, pressureTrackerStats.totalDownloadsStarted); - assertEquals(deserializedStats.totalDownloadsSucceeded, pressureTrackerStats.totalDownloadsSucceeded); assertEquals(deserializedStats.downloadBytesSucceeded, pressureTrackerStats.downloadBytesSucceeded); assertEquals(deserializedStats.downloadBytesStarted, pressureTrackerStats.downloadBytesStarted); - assertEquals(deserializedStats.downloadTimeMovingAverage, pressureTrackerStats.downloadTimeMovingAverage, 0); } } } @@ -698,11 +609,8 @@ private RemoteSegmentTransferTracker constructTracker() { segmentPressureTracker.updateRemoteRefreshTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); segmentPressureTracker.incrementRejectionCount(); segmentPressureTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); - segmentPressureTracker.incrementTotalDownloadsStarted(); - segmentPressureTracker.incrementTotalDownloadsSucceeded(); segmentPressureTracker.addDownloadBytesStarted(50); segmentPressureTracker.addDownloadBytesSucceeded(50); - segmentPressureTracker.addDownloadTime(101); return segmentPressureTracker; } } From 7a2439d989c589f0af956a2758fae767e6f782ff Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:23:08 +0530 Subject: [PATCH 33/36] Fixing UTs and removing repeated isRemoteStoreEnabled checks Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../opensearch/index/shard/IndexShard.java | 23 +++++++++++-------- .../shard/StatsAwareCopyFromRemoteStore.java | 17 +++----------- .../stats/RemoteStoreStatsTestHelper.java | 4 ++-- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index e79c2cb143a38..c7db56c6a811d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4773,8 +4773,7 @@ private String copySegmentFiles( List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); StatsAwareCopyFromRemoteStore statsAwareSegmentCopy = new StatsAwareCopyFromRemoteStore( - remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId), - indexSettings.isRemoteStoreEnabled() + remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId) ); String segmentNFile = null; try { @@ -4787,14 +4786,18 @@ private String copySegmentFiles( for (String file : uploadedSegments.keySet()) { long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - statsAwareSegmentCopy.performCopy( - storeDirectory, - sourceRemoteDirectory, - file, - file, - uploadedSegments.get(file).getLength(), - IOContext.DEFAULT - ); + if (indexSettings.isRemoteStoreEnabled()) { + statsAwareSegmentCopy.performCopy( + storeDirectory, + sourceRemoteDirectory, + file, + file, + uploadedSegments.get(file).getLength(), + IOContext.DEFAULT + ); + } else { + storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); + } downloadedSegments.add(file); } else { skippedSegments.add(file); diff --git a/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java b/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java index bbb1fee090005..72757fe6f4edf 100644 --- a/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java +++ b/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java @@ -23,8 +23,8 @@ public class StatsAwareCopyFromRemoteStore { private final RemoteStoreDownloadStatsListener downloadStatsListener; - public StatsAwareCopyFromRemoteStore(RemoteSegmentTransferTracker downloadStatsTracker, boolean isRemoteStoreEnabled) { - this.downloadStatsListener = new RemoteStoreDownloadStatsListener(downloadStatsTracker, isRemoteStoreEnabled); + public StatsAwareCopyFromRemoteStore(RemoteSegmentTransferTracker downloadStatsTracker) { + this.downloadStatsListener = new RemoteStoreDownloadStatsListener(downloadStatsTracker); } /** @@ -66,13 +66,11 @@ public void performCopy( */ public static class RemoteStoreDownloadStatsListener implements SegmentDownloadListener { private final RemoteSegmentTransferTracker downloadStatsTracker; - private final boolean isRemoteStoreEnabled; private long startTimeInMs; private long fileSize; - public RemoteStoreDownloadStatsListener(RemoteSegmentTransferTracker downloadStatsTracker, boolean isRemoteStoreEnabled) { + public RemoteStoreDownloadStatsListener(RemoteSegmentTransferTracker downloadStatsTracker) { this.downloadStatsTracker = downloadStatsTracker; - this.isRemoteStoreEnabled = isRemoteStoreEnabled; } /** @@ -89,9 +87,6 @@ public void updateFileSize(long fileSize) { */ @Override public void beforeDownload() { - if (!isRemoteStoreEnabled) { - return; - } startTimeInMs = System.currentTimeMillis(); downloadStatsTracker.addDownloadBytesStarted(fileSize); } @@ -105,9 +100,6 @@ public void beforeDownload() { */ @Override public void afterDownload() { - if (!isRemoteStoreEnabled) { - return; - } downloadStatsTracker.addDownloadBytesSucceeded(fileSize); downloadStatsTracker.updateLastDownloadedSegmentSize(fileSize); long currentTimeInMs = System.currentTimeMillis(); @@ -121,9 +113,6 @@ public void afterDownload() { */ @Override public void downloadFailed() { - if (!isRemoteStoreEnabled) { - return; - } downloadStatsTracker.addDownloadBytesFailed(fileSize); } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index ce056cf4d36d2..037db7874afab 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -48,7 +48,7 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewPrimary(ShardId shard 5, 2, 3, - 4, + 0, 0, 0, 0 @@ -82,7 +82,7 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewReplica(ShardId shard 10, 10, 10, - 10 + 0 ); } From 8f8ed298a9a4b879b645d4d4d9abbc25c5c80efd Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Mon, 31 Jul 2023 19:57:37 +0530 Subject: [PATCH 34/36] Moving download stats population to StoreDirectory Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 65 +++--- .../remotestore/stats/RemoteStoreStats.java | 29 ++- .../RemoteRefreshSegmentPressureService.java | 1 + .../remote/RemoteSegmentTransferTracker.java | 164 ++------------- .../opensearch/index/shard/IndexShard.java | 16 +- .../index/shard/SegmentDownloadListener.java | 31 --- .../shard/StatsAwareCopyFromRemoteStore.java | 119 ----------- .../store/DirectoryFileTransferTracker.java | 187 ++++++++++++++++++ .../org/opensearch/index/store/Store.java | 69 ++++++- .../stats/RemoteStoreStatsTestHelper.java | 49 ++--- .../stats/RemoteStoreStatsTests.java | 84 ++++++-- .../RemoteSegmentTransferTrackerTests.java | 136 +++++++------ 12 files changed, 490 insertions(+), 460 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java delete mode 100644 server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java create mode 100644 server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 4e11ccc1cd76f..085ed17976ad7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -72,7 +72,7 @@ public void testStatsResponseFromAllNodes() { assertEquals(1, matches.size()); RemoteSegmentTransferTracker.Stats stats = matches.get(0).getStats(); validateUploadStats(stats); - assertEquals(0, stats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); } // Step 3 - Enable replicas on the existing indices and ensure that download @@ -93,7 +93,7 @@ public void testStatsResponseFromAllNodes() { RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -124,7 +124,7 @@ public void testStatsResponseAllShards() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 3); RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); - assertEquals(0, stats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well @@ -138,7 +138,7 @@ public void testStatsResponseAllShards() { stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -171,7 +171,7 @@ public void testStatsResponseFromLocalNode() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 1); RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); - assertEquals(0, stats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); } changeReplicaCountAndEnsureGreen(1); for (String node : nodes) { @@ -188,7 +188,7 @@ public void testStatsResponseFromLocalNode() { RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -240,7 +240,10 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce .collect(Collectors.toList()) .get(0) .getStats(); - assertTrue(zeroStateReplicaStats.downloadBytesStarted == 0 && zeroStateReplicaStats.downloadBytesSucceeded == 0); + assertTrue( + zeroStateReplicaStats.directoryFileTransferTrackerStats.downloadBytesStarted == 0 + && zeroStateReplicaStats.directoryFileTransferTrackerStats.downloadBytesSucceeded == 0 + ); // Index documents for (int i = 1; i <= randomIntBetween(5, 10); i++) { @@ -267,17 +270,18 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce assertTrue(primaryStats.totalUploadsStarted > 0); assertTrue(primaryStats.totalUploadsSucceeded > 0); assertTrue( - replicaStats.downloadBytesStarted > 0 - && primaryStats.uploadBytesStarted - zeroStatePrimaryStats.uploadBytesStarted == replicaStats.downloadBytesStarted + replicaStats.directoryFileTransferTrackerStats.downloadBytesStarted > 0 + && primaryStats.uploadBytesStarted + - zeroStatePrimaryStats.uploadBytesStarted == replicaStats.directoryFileTransferTrackerStats.downloadBytesStarted ); assertTrue( - replicaStats.downloadBytesSucceeded > 0 + replicaStats.directoryFileTransferTrackerStats.downloadBytesSucceeded > 0 && primaryStats.uploadBytesSucceeded - - zeroStatePrimaryStats.uploadBytesSucceeded == replicaStats.downloadBytesSucceeded + - zeroStatePrimaryStats.uploadBytesSucceeded == replicaStats.directoryFileTransferTrackerStats.downloadBytesSucceeded ); // Assert zero failures assertEquals(0, primaryStats.uploadBytesFailed); - assertEquals(0, replicaStats.downloadBytesFailed); + assertEquals(0, replicaStats.directoryFileTransferTrackerStats.downloadBytesFailed); }, 60, TimeUnit.SECONDS); } } @@ -328,7 +332,10 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr .filter(remoteStoreStats -> !remoteStoreStats.getShardRouting().primary()) .collect(Collectors.toList()); zeroStateReplicaStats.forEach(stats -> { - assertTrue(stats.getStats().downloadBytesStarted == 0 && stats.getStats().downloadBytesSucceeded == 0); + assertTrue( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesStarted == 0 + && stats.getStats().directoryFileTransferTrackerStats.downloadBytesSucceeded == 0 + ); }); int currentNodesInCluster = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); @@ -353,9 +360,9 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr uploadBytesSucceeded = stats.uploadBytesSucceeded; uploadBytesFailed = stats.uploadBytesFailed; } else { - downloadBytesStarted.add(stats.downloadBytesStarted); - downloadBytesSucceeded.add(stats.downloadBytesSucceeded); - downloadBytesFailed.add(stats.downloadBytesFailed); + downloadBytesStarted.add(stats.directoryFileTransferTrackerStats.downloadBytesStarted); + downloadBytesSucceeded.add(stats.directoryFileTransferTrackerStats.downloadBytesSucceeded); + downloadBytesFailed.add(stats.directoryFileTransferTrackerStats.downloadBytesFailed); } } @@ -452,6 +459,7 @@ public void testStatsOnRemoteStoreRestore() throws IOException { // Index some docs to ensure segments being uploaded to remote store indexDocs(); + refresh(INDEX_NAME); // Stop one data node to force the index into a red state internalCluster().stopRandomDataNode(); @@ -461,7 +469,6 @@ public void testStatsOnRemoteStoreRestore() throws IOException { internalCluster().startDataOnlyNode(); // Restore index from remote store - assertAcked(client().admin().indices().prepareClose(INDEX_NAME).get()); client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); // Ensure that the index is green @@ -481,7 +488,10 @@ public void testStatsOnRemoteStoreRestore() throws IOException { assertTrue( segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0 ); - assertTrue(segmentTracker.downloadBytesStarted > 0 && segmentTracker.downloadBytesSucceeded > 0); + assertTrue( + segmentTracker.directoryFileTransferTrackerStats.downloadBytesStarted > 0 + && segmentTracker.directoryFileTransferTrackerStats.downloadBytesSucceeded > 0 + ); }); } @@ -510,7 +520,10 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce && segmentTracker.totalUploadsFailed == 0 ); } else { - assertTrue(segmentTracker.downloadBytesStarted == 0 && segmentTracker.downloadBytesSucceeded == 0); + assertTrue( + segmentTracker.directoryFileTransferTrackerStats.downloadBytesStarted == 0 + && segmentTracker.directoryFileTransferTrackerStats.downloadBytesSucceeded == 0 + ); } }); }, 5, TimeUnit.SECONDS); @@ -564,13 +577,13 @@ private void validateUploadStats(RemoteSegmentTransferTracker.Stats stats) { } private void validateDownloadStats(RemoteSegmentTransferTracker.Stats stats) { - assertTrue(stats.lastDownloadTimestampMs > 0); - assertTrue(stats.downloadBytesStarted > 0); - assertTrue(stats.downloadBytesSucceeded > 0); - assertEquals(stats.downloadBytesFailed, 0); - assertTrue(stats.lastSuccessfulSegmentDownloadBytes > 0); - assertTrue(stats.downloadBytesMovingAverage > 0); - assertTrue(stats.downloadBytesPerSecMovingAverage > 0); + assertTrue(stats.directoryFileTransferTrackerStats.lastDownloadTimestampMs > 0); + assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesStarted > 0); + assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesSucceeded > 0); + assertEquals(stats.directoryFileTransferTrackerStats.downloadBytesFailed, 0); + assertTrue(stats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes > 0); + assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesMovingAverage > 0); + assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage > 0); } // Validate if the shardRouting obtained from cluster state contains the exact same routing object diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index b4541dc66ced7..b046c5e16ca78 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -54,7 +54,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.SEGMENT); builder.startObject(SubFields.DOWNLOAD); // Ensuring that we are not showing 0 metrics to the user - if (remoteSegmentShardStats.downloadBytesStarted != 0) { + if (remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesStarted != 0) { buildDownloadStats(builder); } builder.endObject(); @@ -105,18 +105,21 @@ private void buildUploadStats(XContentBuilder builder) throws IOException { } private void buildDownloadStats(XContentBuilder builder) throws IOException { - builder.field(DownloadStatsFields.LAST_SYNC_TIMESTAMP, remoteSegmentShardStats.lastDownloadTimestampMs); + builder.field( + DownloadStatsFields.LAST_SYNC_TIMESTAMP, + remoteSegmentShardStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs + ); builder.startObject(DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES) - .field(SubFields.STARTED, remoteSegmentShardStats.downloadBytesStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.downloadBytesSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.downloadBytesFailed); + .field(SubFields.STARTED, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesFailed); builder.endObject(); builder.startObject(DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES) - .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.lastSuccessfulSegmentDownloadBytes) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesMovingAverage); + .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesMovingAverage); builder.endObject(); builder.startObject(DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.downloadBytesPerSecMovingAverage); + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage); builder.endObject(); } @@ -211,11 +214,6 @@ static final class DownloadStatsFields { */ static final String LAST_SYNC_TIMESTAMP = "last_sync_timestamp"; - /** - * Total number of sync from the remote store for a specific shard - */ - static final String TOTAL_SYNCS_FROM_REMOTE = "total_syncs_from_remote"; - /** * Total bytes of segment files downloaded from the remote store for a specific shard */ @@ -230,11 +228,6 @@ static final class DownloadStatsFields { * Speed (in bytes/sec) for segment file downloads */ static final String DOWNLOAD_SPEED_IN_BYTES_PER_SEC = "download_speed_in_bytes_per_sec"; - - /** - * Time taken (in millis) for each segment file downloaded - */ - static final String DOWNLOAD_LATENCY_IN_MILLIS = "download_latency_in_millis"; } /** diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java index 75f22054a4c42..6f6364ac3b8a6 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureService.java @@ -76,6 +76,7 @@ public void afterIndexShardCreated(IndexShard indexShard) { shardId, new RemoteSegmentTransferTracker( shardId, + indexShard.store().getDirectoryFileTransferTracker(), pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java index 6f1f48a01c776..cd5d461584f0f 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteSegmentTransferTracker.java @@ -15,6 +15,7 @@ import org.opensearch.common.util.Streak; import org.opensearch.common.util.concurrent.ConcurrentCollections; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.DirectoryFileTransferTracker; import java.io.IOException; import java.util.HashMap; @@ -52,11 +53,6 @@ public class RemoteSegmentTransferTracker { */ private volatile long localRefreshClockTimeMs; - /** - * Time in milliseconds for the last successful segment download - */ - private volatile long lastDownloadTimestampMs; - /** * Sequence number of the most recent remote refresh. */ @@ -87,8 +83,6 @@ public class RemoteSegmentTransferTracker { */ private volatile long lastSuccessfulRemoteRefreshBytes; - private volatile long lastSuccessfulSegmentDownloadBytes; - /** * Cumulative sum of size in bytes of segment files for which upload has started during remote refresh. */ @@ -104,21 +98,6 @@ public class RemoteSegmentTransferTracker { */ private volatile long uploadBytesSucceeded; - /** - * Cumulative sum of size in bytes of segment files for which download has started. - */ - private volatile long downloadBytesStarted; - - /** - * Cumulative sum of size in bytes of segment files for which download has failed. - */ - private volatile long downloadBytesFailed; - - /** - * Cumulative sum of size in bytes of segment files for which download has succeeded. - */ - private volatile long downloadBytesSucceeded; - /** * Cumulative sum of count of remote refreshes that have started. */ @@ -175,12 +154,6 @@ public class RemoteSegmentTransferTracker { */ private final Object uploadBytesMutex = new Object(); - /** - * Provides moving average over the last N total size in bytes of segment files downloaded from the remote store. - * N is window size - */ - private volatile MovingAverage downloadBytesMovingAverageReference; - /** * Provides moving average over the last N upload speed (in bytes/s) of segment files uploaded as part of remote refresh. * N is window size. Wrapped with {@code AtomicReference} for dynamic changes in window size. @@ -190,23 +163,21 @@ public class RemoteSegmentTransferTracker { private final Object uploadBytesPerSecMutex = new Object(); /** - * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the remote store. - * N is window size - */ - private volatile MovingAverage downloadBytesPerSecMovingAverageReference; - - /** - * Provides moving average over the last N overall upload time (in millis) as part of remote refresh.N is window size. + * Provides moving average over the last N overall upload time (in nanos) as part of remote refresh.N is window size. * Wrapped with {@code AtomicReference} for dynamic changes in window size. */ private final AtomicReference uploadTimeMsMovingAverageReference; private final Object uploadTimeMsMutex = new Object(); - private final int SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE = 20; + /** + * {@link org.opensearch.index.store.Store.StoreDirectory} level file transfer tracker, used to show download stats + */ + private final DirectoryFileTransferTracker directoryFileTransferTracker; public RemoteSegmentTransferTracker( ShardId shardId, + DirectoryFileTransferTracker directoryFileTransferTracker, int uploadBytesMovingAverageWindowSize, int uploadBytesPerSecMovingAverageWindowSize, int uploadTimeMsMovingAverageWindowSize @@ -222,9 +193,9 @@ public RemoteSegmentTransferTracker( uploadBytesMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesMovingAverageWindowSize)); uploadBytesPerSecMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadBytesPerSecMovingAverageWindowSize)); uploadTimeMsMovingAverageReference = new AtomicReference<>(new MovingAverage(uploadTimeMsMovingAverageWindowSize)); - downloadBytesMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); - downloadBytesPerSecMovingAverageReference = new MovingAverage(SEGMENT_DOWNLOADS_DEFAULT_WINDOW_SIZE); + latestLocalFileNameLengthMap = new HashMap<>(); + this.directoryFileTransferTracker = directoryFileTransferTracker; } ShardId getShardId() { @@ -253,10 +224,6 @@ public long getLocalRefreshClockTimeMs() { return localRefreshClockTimeMs; } - public long getLastDownloadTimestampMs() { - return lastDownloadTimestampMs; - } - public void updateLocalRefreshTimeMs(long localRefreshTimeMs) { assert localRefreshTimeMs >= this.localRefreshTimeMs : "newLocalRefreshTimeMs=" + localRefreshTimeMs @@ -271,10 +238,6 @@ public void updateLocalRefreshClockTimeMs(long localRefreshClockTimeMs) { this.localRefreshClockTimeMs = localRefreshClockTimeMs; } - public void updateLastDownloadTimestampMs(long downloadTimestampInMs) { - this.lastDownloadTimestampMs = downloadTimestampInMs; - } - long getRemoteRefreshSeqNo() { return remoteRefreshSeqNo; } @@ -355,38 +318,10 @@ public void addUploadBytesSucceeded(long size) { uploadBytesSucceeded += size; } - public long getDownloadBytesStarted() { - return downloadBytesStarted; - } - - public void addDownloadBytesStarted(long size) { - downloadBytesStarted += size; - } - - public long getDownloadBytesFailed() { - return downloadBytesFailed; - } - - public void addDownloadBytesFailed(long size) { - downloadBytesFailed += size; - } - - public long getDownloadBytesSucceeded() { - return downloadBytesSucceeded; - } - - public void addDownloadBytesSucceeded(long size) { - downloadBytesSucceeded += size; - } - public long getInflightUploadBytes() { return uploadBytesStarted - uploadBytesFailed - uploadBytesSucceeded; } - public long getInflightDownloadBytes() { - return downloadBytesStarted - downloadBytesFailed - downloadBytesSucceeded; - } - public long getTotalUploadsStarted() { return totalUploadsStarted; } @@ -495,19 +430,6 @@ void updateUploadBytesMovingAverageWindowSize(int updatedSize) { } } - boolean isDownloadBytesAverageReady() { - return downloadBytesMovingAverageReference.isReady(); - } - - double getDownloadBytesAverage() { - return downloadBytesMovingAverageReference.getAverage(); - } - - public void updateLastDownloadedSegmentSize(long size) { - lastSuccessfulSegmentDownloadBytes = size; - this.downloadBytesMovingAverageReference.record(size); - } - boolean isUploadBytesPerSecAverageReady() { return uploadBytesPerSecMovingAverageReference.get().isReady(); } @@ -522,18 +444,6 @@ public void addUploadBytesPerSec(long bytesPerSec) { } } - boolean isDownloadBytesPerSecAverageReady() { - return downloadBytesPerSecMovingAverageReference.isReady(); - } - - double getDownloadBytesPerSecAverage() { - return downloadBytesPerSecMovingAverageReference.getAverage(); - } - - public void addDownloadBytesPerSec(long bytesPerSec) { - this.downloadBytesPerSecMovingAverageReference.record(bytesPerSec); - } - /** * Updates the window size for data collection of upload bytes per second. This also resets any data collected so far. * @@ -570,21 +480,21 @@ void updateUploadTimeMsMovingAverageWindowSize(int updatedSize) { } } + public DirectoryFileTransferTracker getDirectoryFileTransferTracker() { + return directoryFileTransferTracker; + } + public RemoteSegmentTransferTracker.Stats stats() { return new RemoteSegmentTransferTracker.Stats( shardId, localRefreshClockTimeMs, remoteRefreshClockTimeMs, timeMsLag, - lastDownloadTimestampMs, localRefreshSeqNo, remoteRefreshSeqNo, uploadBytesStarted, uploadBytesSucceeded, uploadBytesFailed, - downloadBytesStarted, - downloadBytesSucceeded, - downloadBytesFailed, totalUploadsStarted, totalUploadsSucceeded, totalUploadsFailed, @@ -594,10 +504,8 @@ public RemoteSegmentTransferTracker.Stats stats() { uploadBytesMovingAverageReference.get().getAverage(), uploadBytesPerSecMovingAverageReference.get().getAverage(), uploadTimeMsMovingAverageReference.get().getAverage(), - lastSuccessfulSegmentDownloadBytes, - downloadBytesMovingAverageReference.getAverage(), - downloadBytesPerSecMovingAverageReference.getAverage(), - getBytesLag() + getBytesLag(), + directoryFileTransferTracker.stats() ); } @@ -612,15 +520,11 @@ public static class Stats implements Writeable { public final long localRefreshClockTimeMs; public final long remoteRefreshClockTimeMs; public final long refreshTimeLagMs; - public final long lastDownloadTimestampMs; public final long localRefreshNumber; public final long remoteRefreshNumber; public final long uploadBytesStarted; public final long uploadBytesFailed; public final long uploadBytesSucceeded; - public final long downloadBytesStarted; - public final long downloadBytesFailed; - public final long downloadBytesSucceeded; public final long totalUploadsStarted; public final long totalUploadsFailed; public final long totalUploadsSucceeded; @@ -630,25 +534,19 @@ public static class Stats implements Writeable { public final double uploadBytesMovingAverage; public final double uploadBytesPerSecMovingAverage; public final double uploadTimeMovingAverage; - public final long lastSuccessfulSegmentDownloadBytes; - public final double downloadBytesMovingAverage; - public final double downloadBytesPerSecMovingAverage; public final long bytesLag; + public final DirectoryFileTransferTracker.Stats directoryFileTransferTrackerStats; public Stats( ShardId shardId, long localRefreshClockTimeMs, long remoteRefreshClockTimeMs, long refreshTimeLagMs, - long lastDownloadTimestampMs, long localRefreshNumber, long remoteRefreshNumber, long uploadBytesStarted, long uploadBytesSucceeded, long uploadBytesFailed, - long downloadBytesStarted, - long downloadBytesSucceeded, - long downloadBytesFailed, long totalUploadsStarted, long totalUploadsSucceeded, long totalUploadsFailed, @@ -658,24 +556,18 @@ public Stats( double uploadBytesMovingAverage, double uploadBytesPerSecMovingAverage, double uploadTimeMovingAverage, - long lastSuccessfulSegmentDownloadBytes, - double downloadBytesMovingAverage, - double downloadBytesPerSecMovingAverage, - long bytesLag + long bytesLag, + DirectoryFileTransferTracker.Stats directoryFileTransferTrackerStats ) { this.shardId = shardId; this.localRefreshClockTimeMs = localRefreshClockTimeMs; this.remoteRefreshClockTimeMs = remoteRefreshClockTimeMs; this.refreshTimeLagMs = refreshTimeLagMs; - this.lastDownloadTimestampMs = lastDownloadTimestampMs; this.localRefreshNumber = localRefreshNumber; this.remoteRefreshNumber = remoteRefreshNumber; this.uploadBytesStarted = uploadBytesStarted; this.uploadBytesFailed = uploadBytesFailed; this.uploadBytesSucceeded = uploadBytesSucceeded; - this.downloadBytesStarted = downloadBytesStarted; - this.downloadBytesFailed = downloadBytesFailed; - this.downloadBytesSucceeded = downloadBytesSucceeded; this.totalUploadsStarted = totalUploadsStarted; this.totalUploadsFailed = totalUploadsFailed; this.totalUploadsSucceeded = totalUploadsSucceeded; @@ -685,10 +577,8 @@ public Stats( this.uploadBytesMovingAverage = uploadBytesMovingAverage; this.uploadBytesPerSecMovingAverage = uploadBytesPerSecMovingAverage; this.uploadTimeMovingAverage = uploadTimeMovingAverage; - this.lastSuccessfulSegmentDownloadBytes = lastSuccessfulSegmentDownloadBytes; - this.downloadBytesMovingAverage = downloadBytesMovingAverage; - this.downloadBytesPerSecMovingAverage = downloadBytesPerSecMovingAverage; this.bytesLag = bytesLag; + this.directoryFileTransferTrackerStats = directoryFileTransferTrackerStats; } public Stats(StreamInput in) throws IOException { @@ -712,13 +602,7 @@ public Stats(StreamInput in) throws IOException { this.uploadBytesPerSecMovingAverage = in.readDouble(); this.uploadTimeMovingAverage = in.readDouble(); this.bytesLag = in.readLong(); - this.lastDownloadTimestampMs = in.readLong(); - this.downloadBytesStarted = in.readLong(); - this.downloadBytesFailed = in.readLong(); - this.downloadBytesSucceeded = in.readLong(); - this.lastSuccessfulSegmentDownloadBytes = in.readLong(); - this.downloadBytesMovingAverage = in.readDouble(); - this.downloadBytesPerSecMovingAverage = in.readDouble(); + this.directoryFileTransferTrackerStats = in.readOptionalWriteable(DirectoryFileTransferTracker.Stats::new); } catch (IOException e) { throw e; } @@ -745,13 +629,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeDouble(uploadBytesPerSecMovingAverage); out.writeDouble(uploadTimeMovingAverage); out.writeLong(bytesLag); - out.writeLong(lastDownloadTimestampMs); - out.writeLong(downloadBytesStarted); - out.writeLong(downloadBytesFailed); - out.writeLong(downloadBytesSucceeded); - out.writeLong(lastSuccessfulSegmentDownloadBytes); - out.writeDouble(downloadBytesMovingAverage); - out.writeDouble(downloadBytesPerSecMovingAverage); + out.writeOptionalWriteable(directoryFileTransferTrackerStats); } } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index aa233057b85f2..841fa6b272951 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4773,9 +4773,6 @@ private String copySegmentFiles( ) throws IOException { List downloadedSegments = new ArrayList<>(); List skippedSegments = new ArrayList<>(); - StatsAwareCopyFromRemoteStore statsAwareSegmentCopy = new StatsAwareCopyFromRemoteStore( - remoteRefreshSegmentPressureService.getRemoteRefreshSegmentTracker(shardId) - ); String segmentNFile = null; try { Set localSegmentFiles = Sets.newHashSet(storeDirectory.listAll()); @@ -4787,18 +4784,7 @@ private String copySegmentFiles( for (String file : uploadedSegments.keySet()) { long checksum = Long.parseLong(uploadedSegments.get(file).getChecksum()); if (overrideLocal || localDirectoryContains(storeDirectory, file, checksum) == false) { - if (indexSettings.isRemoteStoreEnabled()) { - statsAwareSegmentCopy.performCopy( - storeDirectory, - sourceRemoteDirectory, - file, - file, - uploadedSegments.get(file).getLength(), - IOContext.DEFAULT - ); - } else { - storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); - } + storeDirectory.copyFrom(sourceRemoteDirectory, file, file, IOContext.DEFAULT); downloadedSegments.add(file); } else { skippedSegments.add(file); diff --git a/server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java b/server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java deleted file mode 100644 index ac78a4b3e0bec..0000000000000 --- a/server/src/main/java/org/opensearch/index/shard/SegmentDownloadListener.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.shard; - -/** - * Interface with hooks executed on segment download events - * - * @opensearch.internal - */ -public interface SegmentDownloadListener { - /** - * Executed before segment download starts - */ - void beforeDownload(); - - /** - * Executed after segment download completes - */ - void afterDownload(); - - /** - * Executed if segment download fails - */ - void downloadFailed(); -} diff --git a/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java b/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java deleted file mode 100644 index 72757fe6f4edf..0000000000000 --- a/server/src/main/java/org/opensearch/index/shard/StatsAwareCopyFromRemoteStore.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.shard; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.opensearch.index.remote.RemoteSegmentTransferTracker; -import org.opensearch.index.store.RemoteSegmentStoreDirectory; - -import java.io.IOException; - -/** - * Copies over files from a remote store and populates remote store stats accordingly - * - * @opensearch.internal - */ -public class StatsAwareCopyFromRemoteStore { - private final RemoteStoreDownloadStatsListener downloadStatsListener; - - public StatsAwareCopyFromRemoteStore(RemoteSegmentTransferTracker downloadStatsTracker) { - this.downloadStatsListener = new RemoteStoreDownloadStatsListener(downloadStatsTracker); - } - - /** - * Encapsulates the copyFrom method and populates remote store segment download stats - * @param storeDirectory: Destination directory implementation to copy segments to - * @param remoteDirectory: Source {@link RemoteSegmentStoreDirectory} to copy segments from - * @param srcFile: Source file name - * @param destFile: Source file name - * @param fileSize: Size of the file being downloaded - * @param ioContext: {@link IOContext} object to be used in {@link Directory#copyFrom} method - * @throws IOException if an error occurs in copying files - */ - public void performCopy( - Directory storeDirectory, - RemoteSegmentStoreDirectory remoteDirectory, - String srcFile, - String destFile, - long fileSize, - IOContext ioContext - ) throws IOException { - downloadStatsListener.updateFileSize(fileSize); - downloadStatsListener.beforeDownload(); - boolean copySuccessful = false; - try { - storeDirectory.copyFrom(remoteDirectory, srcFile, destFile, ioContext); - copySuccessful = true; - downloadStatsListener.afterDownload(); - } finally { - if (!copySuccessful) { - downloadStatsListener.downloadFailed(); - } - } - } - - /** - * Implementation of {@link SegmentDownloadListener} to populate remote store state related to segment downloads - * - * @opensearch.internal - */ - public static class RemoteStoreDownloadStatsListener implements SegmentDownloadListener { - private final RemoteSegmentTransferTracker downloadStatsTracker; - private long startTimeInMs; - private long fileSize; - - public RemoteStoreDownloadStatsListener(RemoteSegmentTransferTracker downloadStatsTracker) { - this.downloadStatsTracker = downloadStatsTracker; - } - - /** - * Updates the current segment file's size. - * This would be used to update the tracker values - * @param fileSize: Size of the incoming segment file - */ - public void updateFileSize(long fileSize) { - this.fileSize = fileSize; - } - - /** - * Updates the amount of bytes attempted for download - */ - @Override - public void beforeDownload() { - startTimeInMs = System.currentTimeMillis(); - downloadStatsTracker.addDownloadBytesStarted(fileSize); - } - - /** - * Updates - * - The amount of bytes that has been successfully downloaded from the remote store - * - The last successful download completion timestamp - * - The last successfully downloaded segment size - * - The speed of segment download (in bytes/sec) - */ - @Override - public void afterDownload() { - downloadStatsTracker.addDownloadBytesSucceeded(fileSize); - downloadStatsTracker.updateLastDownloadedSegmentSize(fileSize); - long currentTimeInMs = System.currentTimeMillis(); - downloadStatsTracker.updateLastDownloadTimestampMs(currentTimeInMs); - long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); - downloadStatsTracker.addDownloadBytesPerSec((fileSize * 1_000L) / timeTakenInMS); - } - - /** - * Updates the amount of bytes failed in download - */ - @Override - public void downloadFailed() { - downloadStatsTracker.addDownloadBytesFailed(fileSize); - } - } -} diff --git a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java new file mode 100644 index 0000000000000..682907cbfbfd8 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java @@ -0,0 +1,187 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.opensearch.common.util.MovingAverage; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; + +import java.io.IOException; + +/** + * Tracks the amount of bytes transferred between two {@link org.apache.lucene.store.Directory} instances + * + * @opensearch.internal + */ +public class DirectoryFileTransferTracker { + /** + * Cumulative size of files (in bytes) attempted to be copied over from the destination {@link org.apache.lucene.store.Directory} + */ + private volatile long downloadBytesStarted; + + /** + * Cumulative size of files (in bytes) successfully copied over from the destination {@link org.apache.lucene.store.Directory} + */ + private volatile long downloadBytesFailed; + + /** + * Cumulative size of files (in bytes) failed in copying over from the destination {@link org.apache.lucene.store.Directory} + */ + private volatile long downloadBytesSucceeded; + + /** + * Time in milliseconds for the last successful copy operation from the destination {@link org.apache.lucene.store.Directory} + */ + private volatile long lastDownloadTimestampMs; + + /** + * Provides moving average over the last N total size in bytes of files downloaded from the destination {@link org.apache.lucene.store.Directory}. + * N is window size + */ + private volatile MovingAverage downloadBytesMovingAverageReference; + + private volatile long lastSuccessfulSegmentDownloadBytes; + + /** + * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the destination {@link org.apache.lucene.store.Directory}. + * N is window size + */ + private volatile MovingAverage downloadBytesPerSecMovingAverageReference; + + private final int DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE = 20; + + public long getDownloadBytesStarted() { + return downloadBytesStarted; + } + + public void addDownloadBytesStarted(long size) { + downloadBytesStarted += size; + } + + public long getDownloadBytesFailed() { + return downloadBytesFailed; + } + + public void addDownloadBytesFailed(long size) { + downloadBytesFailed += size; + } + + public long getDownloadBytesSucceeded() { + return downloadBytesSucceeded; + } + + public void addDownloadBytesSucceeded(long size) { + downloadBytesSucceeded += size; + } + + public boolean isDownloadBytesPerSecAverageReady() { + return downloadBytesPerSecMovingAverageReference.isReady(); + } + + public double getDownloadBytesPerSecAverage() { + return downloadBytesPerSecMovingAverageReference.getAverage(); + } + + public void addDownloadBytesPerSec(long bytesPerSec) { + this.downloadBytesPerSecMovingAverageReference.record(bytesPerSec); + } + + public boolean isDownloadBytesAverageReady() { + return downloadBytesMovingAverageReference.isReady(); + } + + public double getDownloadBytesAverage() { + return downloadBytesMovingAverageReference.getAverage(); + } + + public void updateLastDownloadedSegmentSize(long size) { + lastSuccessfulSegmentDownloadBytes = size; + this.downloadBytesMovingAverageReference.record(size); + } + + public long getLastDownloadTimestampMs() { + return lastDownloadTimestampMs; + } + + public void updateLastDownloadTimestampMs(long downloadTimestampInMs) { + this.lastDownloadTimestampMs = downloadTimestampInMs; + } + + public DirectoryFileTransferTracker() { + downloadBytesMovingAverageReference = new MovingAverage(DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE); + downloadBytesPerSecMovingAverageReference = new MovingAverage(DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE); + } + + public DirectoryFileTransferTracker.Stats stats() { + return new Stats( + downloadBytesStarted, + downloadBytesFailed, + downloadBytesSucceeded, + lastDownloadTimestampMs, + downloadBytesMovingAverageReference.getAverage(), + lastSuccessfulSegmentDownloadBytes, + downloadBytesPerSecMovingAverageReference.getAverage() + ); + } + + /** + * Represents the tracker's stats presentable to an API. + * + * @opensearch.internal + */ + public static class Stats implements Writeable { + public final long downloadBytesStarted; + public final long downloadBytesFailed; + public final long downloadBytesSucceeded; + public final long lastDownloadTimestampMs; + public final double downloadBytesMovingAverage; + public final long lastSuccessfulSegmentDownloadBytes; + public final double downloadBytesPerSecMovingAverage; + + public Stats( + long downloadBytesStarted, + long downloadBytesFailed, + long downloadBytesSucceeded, + long lastDownloadTimestampMs, + double downloadBytesMovingAverage, + long lastSuccessfulSegmentDownloadBytes, + double downloadBytesPerSecMovingAverage + ) { + this.downloadBytesStarted = downloadBytesStarted; + this.downloadBytesFailed = downloadBytesFailed; + this.downloadBytesSucceeded = downloadBytesSucceeded; + this.lastDownloadTimestampMs = lastDownloadTimestampMs; + this.downloadBytesMovingAverage = downloadBytesMovingAverage; + this.lastSuccessfulSegmentDownloadBytes = lastSuccessfulSegmentDownloadBytes; + this.downloadBytesPerSecMovingAverage = downloadBytesPerSecMovingAverage; + } + + public Stats(StreamInput in) throws IOException { + this.downloadBytesStarted = in.readLong(); + this.downloadBytesFailed = in.readLong(); + this.downloadBytesSucceeded = in.readLong(); + this.lastDownloadTimestampMs = in.readLong(); + this.downloadBytesMovingAverage = in.readDouble(); + this.lastSuccessfulSegmentDownloadBytes = in.readLong(); + this.downloadBytesPerSecMovingAverage = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(downloadBytesStarted); + out.writeLong(downloadBytesFailed); + out.writeLong(downloadBytesSucceeded); + out.writeLong(lastDownloadTimestampMs); + out.writeDouble(downloadBytesMovingAverage); + out.writeLong(lastSuccessfulSegmentDownloadBytes); + out.writeDouble(downloadBytesPerSecMovingAverage); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 8967100d4faf0..3774a8ebd12d1 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -187,6 +187,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref // prevents segment files deletion until the PIT/Scroll expires or is discarded private final ReplicaFileTracker replicaFileTracker; + private final DirectoryFileTransferTracker directoryFileTransferTracker; + private final AbstractRefCounted refCounter = new AbstractRefCounted("store") { @Override protected void closeInternal() { @@ -204,7 +206,12 @@ public Store(ShardId shardId, IndexSettings indexSettings, Directory directory, final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING); logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval); ByteSizeCachingDirectory sizeCachingDir = new ByteSizeCachingDirectory(directory, refreshInterval); - this.directory = new StoreDirectory(sizeCachingDir, Loggers.getLogger("index.store.deletes", shardId)); + this.directoryFileTransferTracker = new DirectoryFileTransferTracker(); + this.directory = new StoreDirectory( + sizeCachingDir, + directoryFileTransferTracker, + Loggers.getLogger("index.store.deletes", shardId) + ); this.shardLock = shardLock; this.onClose = onClose; this.replicaFileTracker = indexSettings.isSegRepEnabled() ? new ReplicaFileTracker() : null; @@ -963,18 +970,27 @@ public void commitSegmentInfos(SegmentInfos latestSegmentInfos, long maxSeqNo, l } } + public DirectoryFileTransferTracker getDirectoryFileTransferTracker() { + return directoryFileTransferTracker; + } + /** * A store directory * * @opensearch.internal */ static final class StoreDirectory extends FilterDirectory { - private final Logger deletesLogger; + public final DirectoryFileTransferTracker directoryFileTransferTracker; - StoreDirectory(ByteSizeCachingDirectory delegateDirectory, Logger deletesLogger) { + StoreDirectory( + ByteSizeCachingDirectory delegateDirectory, + DirectoryFileTransferTracker directoryFileTransferTracker, + Logger deletesLogger + ) { super(delegateDirectory); this.deletesLogger = deletesLogger; + this.directoryFileTransferTracker = directoryFileTransferTracker; } /** Estimate the cumulative size of all files in this directory in bytes. */ @@ -1012,6 +1028,53 @@ public Set getPendingDeletions() throws IOException { // to be removed once fixed in FilterDirectory. return unwrap(this).getPendingDeletions(); } + + @Override + public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { + long fileSize = from.fileLength(src); + beforeDownload(fileSize); + boolean success = false; + try { + long startTime = System.currentTimeMillis(); + super.copyFrom(from, src, dest, context); + success = true; + afterDownload(fileSize, startTime); + } finally { + if (!success) { + downloadFailed(fileSize); + } + } + } + + /** + * Updates the amount of bytes attempted for download + */ + private void beforeDownload(long fileSize) { + directoryFileTransferTracker.addDownloadBytesStarted(fileSize); + } + + /** + * Updates + * - The amount of bytes that has been successfully downloaded from the remote store + * - The last successful download completion timestamp + * - The last successfully downloaded segment size + * - The speed of segment download (in bytes/sec) + */ + private void afterDownload(long fileSize, long startTimeInMs) { + directoryFileTransferTracker.addDownloadBytesSucceeded(fileSize); + directoryFileTransferTracker.updateLastDownloadedSegmentSize(fileSize); + long currentTimeInMs = System.currentTimeMillis(); + directoryFileTransferTracker.updateLastDownloadTimestampMs(currentTimeInMs); + long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); + directoryFileTransferTracker.addDownloadBytesPerSec((fileSize * 1_000L) / timeTakenInMS); + } + + /** + * Updates the amount of bytes failed in download + */ + private void downloadFailed(long fileSize) { + directoryFileTransferTracker.addDownloadBytesFailed(fileSize); + } } /** diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index 037db7874afab..c30224da295e3 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -13,6 +13,7 @@ import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.index.remote.RemoteSegmentTransferTracker; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.DirectoryFileTransferTracker; import java.util.Map; @@ -43,15 +44,9 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewPrimary(ShardId shard 5, 5, 0, - 3, - 2, - 5, - 2, - 3, 0, 0, - 0, - 0 + createZeroDirectoryFileTransferStats() ); } @@ -67,9 +62,6 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewReplica(ShardId shard 0, 0, 0, - 10, - 10, - 0, 0, 0, 0, @@ -79,10 +71,7 @@ static RemoteSegmentTransferTracker.Stats createStatsForNewReplica(ShardId shard 0, 0, 0, - 10, - 10, - 10, - 0 + createSampleDirectoryFileTransferStats() ); } @@ -107,16 +96,18 @@ static RemoteSegmentTransferTracker.Stats createStatsForRemoteStoreRestoredPrima 0, 0, 100, - 12, - 2, - 2, - 10, - 10, - 10, - 10 + createSampleDirectoryFileTransferStats() ); } + static DirectoryFileTransferTracker.Stats createSampleDirectoryFileTransferStats() { + return new DirectoryFileTransferTracker.Stats(10, 0, 10, 12345, 5, 5, 5); + } + + static DirectoryFileTransferTracker.Stats createZeroDirectoryFileTransferStats() { + return new DirectoryFileTransferTracker.Stats(0, 0, 0, 0, 0, 0, 0); + } + static ShardRouting createShardRouting(ShardId shardId, boolean isPrimary) { return TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(4), isPrimary, ShardRoutingState.STARTED); } @@ -143,46 +134,46 @@ static void compareStatsResponse( Map segmentDownloads = ((Map) segment.get(RemoteStoreStats.SubFields.DOWNLOAD)); Map segmentUploads = ((Map) segment.get(RemoteStoreStats.SubFields.UPLOAD)); - if (statsTracker.downloadBytesStarted != 0) { + if (statsTracker.directoryFileTransferTrackerStats.downloadBytesStarted != 0) { assertEquals( segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_SYNC_TIMESTAMP), - (int) statsTracker.lastDownloadTimestampMs + (int) statsTracker.directoryFileTransferTrackerStats.lastDownloadTimestampMs ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.STARTED ), - (int) statsTracker.downloadBytesStarted + (int) statsTracker.directoryFileTransferTrackerStats.downloadBytesStarted ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.SUCCEEDED ), - (int) statsTracker.downloadBytesSucceeded + (int) statsTracker.directoryFileTransferTrackerStats.downloadBytesSucceeded ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.FAILED ), - (int) statsTracker.downloadBytesFailed + (int) statsTracker.directoryFileTransferTrackerStats.downloadBytesFailed ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.LAST_SUCCESSFUL ), - (int) statsTracker.lastSuccessfulSegmentDownloadBytes + (int) statsTracker.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - statsTracker.downloadBytesMovingAverage + statsTracker.directoryFileTransferTrackerStats.downloadBytesMovingAverage ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - statsTracker.downloadBytesPerSecMovingAverage + statsTracker.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage ); } else { assertTrue(segmentDownloads.isEmpty()); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 05bd642a34587..044f21a3869af 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -103,11 +103,11 @@ public void testSerializationForPrimaryShard() throws Exception { assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); - assertEquals(0, deserializedStats.downloadBytesStarted); - assertEquals(0, deserializedStats.downloadBytesFailed); - assertEquals(0, deserializedStats.downloadBytesSucceeded); - assertEquals(0, deserializedStats.lastSuccessfulSegmentDownloadBytes); - assertEquals(0, deserializedStats.lastDownloadTimestampMs); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.downloadBytesFailed); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs); } } } @@ -131,13 +131,36 @@ public void testSerializationForReplicaShard() throws Exception { assertEquals(0, deserializedStats.rejectionCount); assertEquals(0, deserializedStats.consecutiveFailuresCount); assertEquals(0, deserializedStats.bytesLag); - assertEquals(stats.getStats().downloadBytesStarted, deserializedStats.downloadBytesStarted); - assertEquals(stats.getStats().downloadBytesFailed, deserializedStats.downloadBytesFailed); - assertEquals(stats.getStats().downloadBytesSucceeded, deserializedStats.downloadBytesSucceeded); - assertEquals(stats.getStats().lastSuccessfulSegmentDownloadBytes, deserializedStats.lastSuccessfulSegmentDownloadBytes); - assertEquals(stats.getStats().lastDownloadTimestampMs, deserializedStats.lastDownloadTimestampMs); - assertEquals(stats.getStats().downloadBytesPerSecMovingAverage, deserializedStats.downloadBytesPerSecMovingAverage, 0); - assertEquals(stats.getStats().downloadBytesMovingAverage, deserializedStats.downloadBytesMovingAverage, 0); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesStarted, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesFailed, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesFailed + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesSucceeded, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes, + deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.lastDownloadTimestampMs, + deserializedStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + 0 + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesMovingAverage, + 0 + ); } } } @@ -164,13 +187,36 @@ public void testSerializationForRemoteStoreRestoredPrimaryShard() throws Excepti assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); - assertEquals(stats.getStats().downloadBytesStarted, deserializedStats.downloadBytesStarted); - assertEquals(stats.getStats().downloadBytesFailed, deserializedStats.downloadBytesFailed); - assertEquals(stats.getStats().downloadBytesSucceeded, deserializedStats.downloadBytesSucceeded); - assertEquals(stats.getStats().lastSuccessfulSegmentDownloadBytes, deserializedStats.lastSuccessfulSegmentDownloadBytes); - assertEquals(stats.getStats().lastDownloadTimestampMs, deserializedStats.lastDownloadTimestampMs); - assertEquals(stats.getStats().downloadBytesPerSecMovingAverage, deserializedStats.downloadBytesPerSecMovingAverage, 0); - assertEquals(stats.getStats().downloadBytesMovingAverage, deserializedStats.downloadBytesMovingAverage, 0); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesStarted, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesFailed, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesFailed + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesSucceeded, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes, + deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.lastDownloadTimestampMs, + deserializedStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + 0 + ); + assertEquals( + stats.getStats().directoryFileTransferTrackerStats.downloadBytesMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.downloadBytesMovingAverage, + 0 + ); } } } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java index 0e4c87d604813..8fea8f98d940b 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java @@ -14,6 +14,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -36,6 +37,8 @@ public class RemoteSegmentTransferTrackerTests extends OpenSearchTestCase { private RemoteSegmentTransferTracker pressureTracker; + private DirectoryFileTransferTracker directoryFileTransferTracker; + @Override public void setUp() throws Exception { super.setUp(); @@ -51,6 +54,7 @@ public void setUp() throws Exception { mock(RemoteRefreshSegmentPressureService.class) ); shardId = new ShardId("index", "uuid", 0); + directoryFileTransferTracker = new DirectoryFileTransferTracker(); } @Override @@ -62,6 +66,7 @@ public void tearDown() throws Exception { public void testGetShardId() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -72,6 +77,7 @@ public void testGetShardId() { public void testUpdateLocalRefreshSeqNo() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -84,6 +90,7 @@ public void testUpdateLocalRefreshSeqNo() { public void testUpdateRemoteRefreshSeqNo() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -96,6 +103,7 @@ public void testUpdateRemoteRefreshSeqNo() { public void testUpdateLocalRefreshTimeMs() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -108,6 +116,7 @@ public void testUpdateLocalRefreshTimeMs() { public void testUpdateRemoteRefreshTimeMs() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -120,18 +129,20 @@ public void testUpdateRemoteRefreshTimeMs() { public void testLastDownloadTimestampMs() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); long currentTimeInMs = System.currentTimeMillis(); - pressureTracker.updateLastDownloadTimestampMs(currentTimeInMs); - assertEquals(currentTimeInMs, pressureTracker.getLastDownloadTimestampMs()); + pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadTimestampMs(currentTimeInMs); + assertEquals(currentTimeInMs, pressureTracker.getDirectoryFileTransferTracker().getLastDownloadTimestampMs()); } public void testComputeSeqNoLagOnUpdate() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -147,6 +158,7 @@ public void testComputeSeqNoLagOnUpdate() { public void testComputeTimeLagOnUpdate() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -164,6 +176,7 @@ public void testComputeTimeLagOnUpdate() { public void testAddUploadBytesStarted() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -179,6 +192,7 @@ public void testAddUploadBytesStarted() { public void testAddUploadBytesFailed() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -194,6 +208,7 @@ public void testAddUploadBytesFailed() { public void testAddUploadBytesSucceeded() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -209,51 +224,55 @@ public void testAddUploadBytesSucceeded() { public void testAddDownloadBytesStarted() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); - pressureTracker.addDownloadBytesStarted(bytesToAdd); - assertEquals(bytesToAdd, pressureTracker.getDownloadBytesStarted()); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesStarted(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesStarted()); long moreBytesToAdd = randomLongBetween(1000, 10000); - pressureTracker.addDownloadBytesStarted(moreBytesToAdd); - assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDownloadBytesStarted()); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesStarted(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesStarted()); } public void testAddDownloadBytesFailed() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); - pressureTracker.addDownloadBytesFailed(bytesToAdd); - assertEquals(bytesToAdd, pressureTracker.getDownloadBytesFailed()); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesFailed(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesFailed()); long moreBytesToAdd = randomLongBetween(1000, 10000); - pressureTracker.addDownloadBytesFailed(moreBytesToAdd); - assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDownloadBytesFailed()); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesFailed(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesFailed()); } public void testAddDownloadBytesSucceeded() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); - pressureTracker.addDownloadBytesSucceeded(bytesToAdd); - assertEquals(bytesToAdd, pressureTracker.getDownloadBytesSucceeded()); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesSucceeded(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesSucceeded()); long moreBytesToAdd = randomLongBetween(1000, 10000); - pressureTracker.addDownloadBytesSucceeded(moreBytesToAdd); - assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDownloadBytesSucceeded()); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesSucceeded(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesSucceeded()); } public void testGetInflightUploadBytes() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -267,25 +286,10 @@ public void testGetInflightUploadBytes() { assertEquals(bytesStarted - bytesSucceeded - bytesFailed, pressureTracker.getInflightUploadBytes()); } - public void testGetInFlightDownloadBytes() { - pressureTracker = new RemoteSegmentTransferTracker( - shardId, - pressureSettings.getUploadBytesMovingAverageWindowSize(), - pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), - pressureSettings.getUploadTimeMovingAverageWindowSize() - ); - long bytesStarted = randomLongBetween(10000, 100000); - long bytesSucceeded = randomLongBetween(1000, 10000); - long bytesFailed = randomLongBetween(100, 1000); - pressureTracker.addDownloadBytesStarted(bytesStarted); - pressureTracker.addDownloadBytesSucceeded(bytesSucceeded); - pressureTracker.addDownloadBytesFailed(bytesFailed); - assertEquals(bytesStarted - bytesSucceeded - bytesFailed, pressureTracker.getInflightDownloadBytes()); - } - public void testIncrementTotalUploadsStarted() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -299,6 +303,7 @@ public void testIncrementTotalUploadsStarted() { public void testIncrementTotalUploadsFailed() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -312,6 +317,7 @@ public void testIncrementTotalUploadsFailed() { public void testIncrementTotalUploadSucceeded() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -325,6 +331,7 @@ public void testIncrementTotalUploadSucceeded() { public void testGetInflightUploads() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -342,6 +349,7 @@ public void testGetInflightUploads() { public void testIncrementRejectionCount() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -355,6 +363,7 @@ public void testIncrementRejectionCount() { public void testGetConsecutiveFailureCount() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -370,6 +379,7 @@ public void testGetConsecutiveFailureCount() { public void testComputeBytesLag() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -399,6 +409,7 @@ public void testComputeBytesLag() { public void testIsUploadBytesAverageReady() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -426,6 +437,7 @@ public void testIsUploadBytesAverageReady() { public void testIsUploadBytesPerSecAverageReady() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -453,6 +465,7 @@ public void testIsUploadBytesPerSecAverageReady() { public void testIsUploadTimeMsAverageReady() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -480,55 +493,57 @@ public void testIsUploadTimeMsAverageReady() { public void testIsDownloadBytesAverageReady() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); - assertFalse(pressureTracker.isDownloadBytesAverageReady()); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesAverageReady()); long sum = 0; for (int i = 1; i < 20; i++) { - pressureTracker.updateLastDownloadedSegmentSize(i); + pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadedSegmentSize(i); sum += i; - assertFalse(pressureTracker.isDownloadBytesAverageReady()); - assertEquals((double) sum / i, pressureTracker.getDownloadBytesAverage(), 0.0d); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesAverage(), 0.0d); } - pressureTracker.updateLastDownloadedSegmentSize(20); + pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadedSegmentSize(20); sum += 20; - assertTrue(pressureTracker.isDownloadBytesAverageReady()); - assertEquals((double) sum / 20, pressureTracker.getDownloadBytesAverage(), 0.0d); + assertTrue(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesAverage(), 0.0d); - pressureTracker.updateLastDownloadedSegmentSize(100); + pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadedSegmentSize(100); sum = sum + 100 - 1; - assertEquals((double) sum / 20, pressureTracker.getDownloadBytesAverage(), 0.0d); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesAverage(), 0.0d); } public void testIsDownloadBytesPerSecAverageReady() { pressureTracker = new RemoteSegmentTransferTracker( shardId, + directoryFileTransferTracker, pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); - assertFalse(pressureTracker.isDownloadBytesPerSecAverageReady()); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesPerSecAverageReady()); long sum = 0; for (int i = 1; i < 20; i++) { - pressureTracker.addDownloadBytesPerSec(i); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(i); sum += i; - assertFalse(pressureTracker.isDownloadBytesPerSecAverageReady()); - assertEquals((double) sum / i, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesPerSecAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesPerSecAverage(), 0.0d); } - pressureTracker.addDownloadBytesPerSec(20); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(20); sum += 20; - assertTrue(pressureTracker.isDownloadBytesPerSecAverageReady()); - assertEquals((double) sum / 20, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); + assertTrue(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesPerSecAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesPerSecAverage(), 0.0d); - pressureTracker.addDownloadBytesPerSec(100); + pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(100); sum = sum + 100 - 1; - assertEquals((double) sum / 20, pressureTracker.getDownloadBytesPerSecAverage(), 0.0d); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesPerSecAverage(), 0.0d); } /** @@ -553,9 +568,6 @@ public void testStatsObjectCreation() { assertEquals(pressureTracker.getTotalUploadsStarted(), (int) pressureTrackerStats.totalUploadsStarted); assertEquals(pressureTracker.getTotalUploadsSucceeded(), (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals(pressureTracker.getTotalUploadsFailed(), (int) pressureTrackerStats.totalUploadsFailed); - assertEquals(pressureTracker.getLastDownloadTimestampMs(), pressureTrackerStats.lastDownloadTimestampMs); - assertEquals(pressureTracker.getDownloadBytesStarted(), pressureTrackerStats.downloadBytesStarted); - assertEquals(pressureTracker.getDownloadBytesSucceeded(), pressureTrackerStats.downloadBytesStarted); } /** @@ -589,9 +601,18 @@ public void testStatsObjectCreationViaStream() throws IOException { assertEquals((int) deserializedStats.totalUploadsStarted, (int) pressureTrackerStats.totalUploadsStarted); assertEquals((int) deserializedStats.totalUploadsSucceeded, (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals((int) deserializedStats.totalUploadsFailed, (int) pressureTrackerStats.totalUploadsFailed); - assertEquals(deserializedStats.lastDownloadTimestampMs, pressureTrackerStats.lastDownloadTimestampMs); - assertEquals(deserializedStats.downloadBytesSucceeded, pressureTrackerStats.downloadBytesSucceeded); - assertEquals(deserializedStats.downloadBytesStarted, pressureTrackerStats.downloadBytesStarted); + assertEquals( + (int) deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted, + (int) pressureTrackerStats.directoryFileTransferTrackerStats.downloadBytesStarted + ); + assertEquals( + (int) deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded, + (int) pressureTrackerStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + ); + assertEquals( + (int) deserializedStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + (int) pressureTrackerStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage + ); } } } @@ -599,6 +620,7 @@ public void testStatsObjectCreationViaStream() throws IOException { private RemoteSegmentTransferTracker constructTracker() { RemoteSegmentTransferTracker segmentPressureTracker = new RemoteSegmentTransferTracker( shardId, + new DirectoryFileTransferTracker(), pressureSettings.getUploadBytesMovingAverageWindowSize(), pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() @@ -608,9 +630,9 @@ private RemoteSegmentTransferTracker constructTracker() { segmentPressureTracker.addUploadBytes(99); segmentPressureTracker.updateRemoteRefreshTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); segmentPressureTracker.incrementRejectionCount(); - segmentPressureTracker.updateLastDownloadTimestampMs(System.currentTimeMillis()); - segmentPressureTracker.addDownloadBytesStarted(50); - segmentPressureTracker.addDownloadBytesSucceeded(50); + segmentPressureTracker.getDirectoryFileTransferTracker().addDownloadBytesStarted(10); + segmentPressureTracker.getDirectoryFileTransferTracker().addDownloadBytesSucceeded(10); + segmentPressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(5); return segmentPressureTracker; } } From c54cd011d038c37fa462ab4d5e5afd03aefc292e Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Mon, 31 Jul 2023 20:28:43 +0530 Subject: [PATCH 35/36] Fixing pressure service UTs and updating tracker javadocs Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../java/org/opensearch/index/shard/IndexShard.java | 1 - .../index/store/DirectoryFileTransferTracker.java | 12 ++++++------ .../main/java/org/opensearch/index/store/Store.java | 6 +++--- .../RemoteRefreshSegmentPressureServiceTests.java | 4 +++- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 841fa6b272951..e09a218ccf83b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -739,7 +739,6 @@ public void onFailure(Exception e) { + this.shardRouting; shardStateUpdated.countDown(); } - if (currentRouting.active() == false && newRouting.active()) { indexEventListener.afterIndexShardStarted(this); } diff --git a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java index 682907cbfbfd8..36aed03ee8b8b 100644 --- a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java @@ -22,27 +22,27 @@ */ public class DirectoryFileTransferTracker { /** - * Cumulative size of files (in bytes) attempted to be copied over from the destination {@link org.apache.lucene.store.Directory} + * Cumulative size of files (in bytes) attempted to be copied over from the source {@link org.apache.lucene.store.Directory} */ private volatile long downloadBytesStarted; /** - * Cumulative size of files (in bytes) successfully copied over from the destination {@link org.apache.lucene.store.Directory} + * Cumulative size of files (in bytes) successfully copied over from the source {@link org.apache.lucene.store.Directory} */ private volatile long downloadBytesFailed; /** - * Cumulative size of files (in bytes) failed in copying over from the destination {@link org.apache.lucene.store.Directory} + * Cumulative size of files (in bytes) failed in copying over from the source {@link org.apache.lucene.store.Directory} */ private volatile long downloadBytesSucceeded; /** - * Time in milliseconds for the last successful copy operation from the destination {@link org.apache.lucene.store.Directory} + * Time in milliseconds for the last successful copy operation from the source {@link org.apache.lucene.store.Directory} */ private volatile long lastDownloadTimestampMs; /** - * Provides moving average over the last N total size in bytes of files downloaded from the destination {@link org.apache.lucene.store.Directory}. + * Provides moving average over the last N total size in bytes of files downloaded from the source {@link org.apache.lucene.store.Directory}. * N is window size */ private volatile MovingAverage downloadBytesMovingAverageReference; @@ -50,7 +50,7 @@ public class DirectoryFileTransferTracker { private volatile long lastSuccessfulSegmentDownloadBytes; /** - * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the destination {@link org.apache.lucene.store.Directory}. + * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the source {@link org.apache.lucene.store.Directory}. * N is window size */ private volatile MovingAverage downloadBytesPerSecMovingAverageReference; diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 3774a8ebd12d1..91b74b11093f3 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -1055,10 +1055,10 @@ private void beforeDownload(long fileSize) { /** * Updates - * - The amount of bytes that has been successfully downloaded from the remote store + * - The amount of bytes that has been successfully downloaded from the source store * - The last successful download completion timestamp - * - The last successfully downloaded segment size - * - The speed of segment download (in bytes/sec) + * - The last successfully downloaded file + * - Download speed (in bytes/sec) */ private void afterDownload(long fileSize, long startTimeInMs) { directoryFileTransferTracker.addDownloadBytesSucceeded(fileSize); diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java index 30c1cb1f4643e..7b180a71f3bab 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteRefreshSegmentPressureServiceTests.java @@ -16,6 +16,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.IndexShard; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.store.Store; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -152,10 +153,11 @@ public void testValidateSegmentUploadLag() { private static IndexShard createIndexShard(ShardId shardId, boolean remoteStoreEnabled) { Settings settings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, String.valueOf(remoteStoreEnabled)).build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test_index", settings); + Store store = mock(Store.class); IndexShard indexShard = mock(IndexShard.class); when(indexShard.indexSettings()).thenReturn(indexSettings); when(indexShard.shardId()).thenReturn(shardId); + when(indexShard.store()).thenReturn(store); return indexShard; } - } From 508eb22a4f6d6187c6f2b5e898b1ba7298a241fb Mon Sep 17 00:00:00 2001 From: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:28:50 +0530 Subject: [PATCH 36/36] Addressing comments Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com> --- .../remotestore/RemoteStoreStatsIT.java | 71 ++++--- .../remotestore/stats/RemoteStoreStats.java | 16 +- .../store/DirectoryFileTransferTracker.java | 178 +++++++++--------- .../org/opensearch/index/store/Store.java | 35 ++-- .../stats/RemoteStoreStatsTestHelper.java | 16 +- .../stats/RemoteStoreStatsTests.java | 66 +++---- .../RemoteSegmentTransferTrackerTests.java | 82 ++++---- 7 files changed, 230 insertions(+), 234 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 085ed17976ad7..840e3a07ed255 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -72,7 +72,7 @@ public void testStatsResponseFromAllNodes() { assertEquals(1, matches.size()); RemoteSegmentTransferTracker.Stats stats = matches.get(0).getStats(); validateUploadStats(stats); - assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.transferredBytesStarted); } // Step 3 - Enable replicas on the existing indices and ensure that download @@ -93,7 +93,7 @@ public void testStatsResponseFromAllNodes() { RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.transferredBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -124,7 +124,7 @@ public void testStatsResponseAllShards() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 3); RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); - assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.transferredBytesStarted); // Step 3 - Enable replicas on the existing indices and ensure that download // stats are being populated as well @@ -138,7 +138,7 @@ public void testStatsResponseAllShards() { stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.transferredBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -171,7 +171,7 @@ public void testStatsResponseFromLocalNode() { assertTrue(response.getRemoteStoreStats() != null && response.getRemoteStoreStats().length == 1); RemoteSegmentTransferTracker.Stats stats = response.getRemoteStoreStats()[0].getStats(); validateUploadStats(stats); - assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.transferredBytesStarted); } changeReplicaCountAndEnsureGreen(1); for (String node : nodes) { @@ -188,7 +188,7 @@ public void testStatsResponseFromLocalNode() { RemoteSegmentTransferTracker.Stats stats = stat.getStats(); if (routing.primary()) { validateUploadStats(stats); - assertEquals(0, stats.directoryFileTransferTrackerStats.downloadBytesStarted); + assertEquals(0, stats.directoryFileTransferTrackerStats.transferredBytesStarted); } else { validateDownloadStats(stats); assertEquals(0, stats.totalUploadsStarted); @@ -241,8 +241,8 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce .get(0) .getStats(); assertTrue( - zeroStateReplicaStats.directoryFileTransferTrackerStats.downloadBytesStarted == 0 - && zeroStateReplicaStats.directoryFileTransferTrackerStats.downloadBytesSucceeded == 0 + zeroStateReplicaStats.directoryFileTransferTrackerStats.transferredBytesStarted == 0 + && zeroStateReplicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded == 0 ); // Index documents @@ -270,18 +270,18 @@ public void testDownloadStatsCorrectnessSinglePrimarySingleReplica() throws Exce assertTrue(primaryStats.totalUploadsStarted > 0); assertTrue(primaryStats.totalUploadsSucceeded > 0); assertTrue( - replicaStats.directoryFileTransferTrackerStats.downloadBytesStarted > 0 + replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted > 0 && primaryStats.uploadBytesStarted - - zeroStatePrimaryStats.uploadBytesStarted == replicaStats.directoryFileTransferTrackerStats.downloadBytesStarted + - zeroStatePrimaryStats.uploadBytesStarted == replicaStats.directoryFileTransferTrackerStats.transferredBytesStarted ); assertTrue( - replicaStats.directoryFileTransferTrackerStats.downloadBytesSucceeded > 0 + replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded > 0 && primaryStats.uploadBytesSucceeded - - zeroStatePrimaryStats.uploadBytesSucceeded == replicaStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + - zeroStatePrimaryStats.uploadBytesSucceeded == replicaStats.directoryFileTransferTrackerStats.transferredBytesSucceeded ); // Assert zero failures assertEquals(0, primaryStats.uploadBytesFailed); - assertEquals(0, replicaStats.directoryFileTransferTrackerStats.downloadBytesFailed); + assertEquals(0, replicaStats.directoryFileTransferTrackerStats.transferredBytesFailed); }, 60, TimeUnit.SECONDS); } } @@ -333,8 +333,8 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr .collect(Collectors.toList()); zeroStateReplicaStats.forEach(stats -> { assertTrue( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesStarted == 0 - && stats.getStats().directoryFileTransferTrackerStats.downloadBytesSucceeded == 0 + stats.getStats().directoryFileTransferTrackerStats.transferredBytesStarted == 0 + && stats.getStats().directoryFileTransferTrackerStats.transferredBytesSucceeded == 0 ); }); @@ -360,9 +360,9 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr uploadBytesSucceeded = stats.uploadBytesSucceeded; uploadBytesFailed = stats.uploadBytesFailed; } else { - downloadBytesStarted.add(stats.directoryFileTransferTrackerStats.downloadBytesStarted); - downloadBytesSucceeded.add(stats.directoryFileTransferTrackerStats.downloadBytesSucceeded); - downloadBytesFailed.add(stats.directoryFileTransferTrackerStats.downloadBytesFailed); + downloadBytesStarted.add(stats.directoryFileTransferTrackerStats.transferredBytesStarted); + downloadBytesSucceeded.add(stats.directoryFileTransferTrackerStats.transferredBytesSucceeded); + downloadBytesFailed.add(stats.directoryFileTransferTrackerStats.transferredBytesFailed); } } @@ -469,7 +469,10 @@ public void testStatsOnRemoteStoreRestore() throws IOException { internalCluster().startDataOnlyNode(); // Restore index from remote store - client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(true), PlainActionFuture.newFuture()); // Ensure that the index is green ensureGreen(INDEX_NAME); @@ -477,20 +480,16 @@ public void testStatsOnRemoteStoreRestore() throws IOException { // Index some more docs to force segment uploads to remote store indexDocs(); - RemoteStoreStats[] remoteStoreStats = client().admin() - .cluster() - .prepareRemoteStoreStats(INDEX_NAME, "0") - .get() - .getRemoteStoreStats(); - Arrays.stream(remoteStoreStats).forEach(statObject -> { + RemoteStoreStatsResponse remoteStoreStatsResponse = client().admin().cluster().prepareRemoteStoreStats(INDEX_NAME, "0").get(); + Arrays.stream(remoteStoreStatsResponse.getRemoteStoreStats()).forEach(statObject -> { RemoteSegmentTransferTracker.Stats segmentTracker = statObject.getStats(); // Assert that we have both upload and download stats for the index assertTrue( segmentTracker.totalUploadsStarted > 0 && segmentTracker.totalUploadsSucceeded > 0 && segmentTracker.totalUploadsFailed == 0 ); assertTrue( - segmentTracker.directoryFileTransferTrackerStats.downloadBytesStarted > 0 - && segmentTracker.directoryFileTransferTrackerStats.downloadBytesSucceeded > 0 + segmentTracker.directoryFileTransferTrackerStats.transferredBytesStarted > 0 + && segmentTracker.directoryFileTransferTrackerStats.transferredBytesSucceeded > 0 ); }); } @@ -521,8 +520,8 @@ public void testNonZeroPrimaryStatsOnNewlyCreatedIndexWithZeroDocs() throws Exce ); } else { assertTrue( - segmentTracker.directoryFileTransferTrackerStats.downloadBytesStarted == 0 - && segmentTracker.directoryFileTransferTrackerStats.downloadBytesSucceeded == 0 + segmentTracker.directoryFileTransferTrackerStats.transferredBytesStarted == 0 + && segmentTracker.directoryFileTransferTrackerStats.transferredBytesSucceeded == 0 ); } }); @@ -577,13 +576,13 @@ private void validateUploadStats(RemoteSegmentTransferTracker.Stats stats) { } private void validateDownloadStats(RemoteSegmentTransferTracker.Stats stats) { - assertTrue(stats.directoryFileTransferTrackerStats.lastDownloadTimestampMs > 0); - assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesStarted > 0); - assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesSucceeded > 0); - assertEquals(stats.directoryFileTransferTrackerStats.downloadBytesFailed, 0); - assertTrue(stats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes > 0); - assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesMovingAverage > 0); - assertTrue(stats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage > 0); + assertTrue(stats.directoryFileTransferTrackerStats.lastTransferTimestampMs > 0); + assertTrue(stats.directoryFileTransferTrackerStats.transferredBytesStarted > 0); + assertTrue(stats.directoryFileTransferTrackerStats.transferredBytesSucceeded > 0); + assertEquals(stats.directoryFileTransferTrackerStats.transferredBytesFailed, 0); + assertTrue(stats.directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes > 0); + assertTrue(stats.directoryFileTransferTrackerStats.transferredBytesMovingAverage > 0); + assertTrue(stats.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage > 0); } // Validate if the shardRouting obtained from cluster state contains the exact same routing object diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java index b046c5e16ca78..6b4c9a26ab19b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStats.java @@ -54,7 +54,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.SEGMENT); builder.startObject(SubFields.DOWNLOAD); // Ensuring that we are not showing 0 metrics to the user - if (remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesStarted != 0) { + if (remoteSegmentShardStats.directoryFileTransferTrackerStats.transferredBytesStarted != 0) { buildDownloadStats(builder); } builder.endObject(); @@ -107,19 +107,19 @@ private void buildUploadStats(XContentBuilder builder) throws IOException { private void buildDownloadStats(XContentBuilder builder) throws IOException { builder.field( DownloadStatsFields.LAST_SYNC_TIMESTAMP, - remoteSegmentShardStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs + remoteSegmentShardStats.directoryFileTransferTrackerStats.lastTransferTimestampMs ); builder.startObject(DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES) - .field(SubFields.STARTED, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesStarted) - .field(SubFields.SUCCEEDED, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesSucceeded) - .field(SubFields.FAILED, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesFailed); + .field(SubFields.STARTED, remoteSegmentShardStats.directoryFileTransferTrackerStats.transferredBytesStarted) + .field(SubFields.SUCCEEDED, remoteSegmentShardStats.directoryFileTransferTrackerStats.transferredBytesSucceeded) + .field(SubFields.FAILED, remoteSegmentShardStats.directoryFileTransferTrackerStats.transferredBytesFailed); builder.endObject(); builder.startObject(DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES) - .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesMovingAverage); + .field(SubFields.LAST_SUCCESSFUL, remoteSegmentShardStats.directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes) + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.directoryFileTransferTrackerStats.transferredBytesMovingAverage); builder.endObject(); builder.startObject(DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC) - .field(SubFields.MOVING_AVG, remoteSegmentShardStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage); + .field(SubFields.MOVING_AVG, remoteSegmentShardStats.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage); builder.endObject(); } diff --git a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java index 36aed03ee8b8b..7e0e231d7bad9 100644 --- a/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java +++ b/server/src/main/java/org/opensearch/index/store/DirectoryFileTransferTracker.java @@ -22,112 +22,120 @@ */ public class DirectoryFileTransferTracker { /** - * Cumulative size of files (in bytes) attempted to be copied over from the source {@link org.apache.lucene.store.Directory} + * Cumulative size of files (in bytes) attempted to be transferred over from the source {@link org.apache.lucene.store.Directory} */ - private volatile long downloadBytesStarted; + private volatile long transferredBytesStarted; /** - * Cumulative size of files (in bytes) successfully copied over from the source {@link org.apache.lucene.store.Directory} + * Cumulative size of files (in bytes) successfully transferred over from the source {@link org.apache.lucene.store.Directory} */ - private volatile long downloadBytesFailed; + private volatile long transferredBytesFailed; /** - * Cumulative size of files (in bytes) failed in copying over from the source {@link org.apache.lucene.store.Directory} + * Cumulative size of files (in bytes) failed in transfer over from the source {@link org.apache.lucene.store.Directory} */ - private volatile long downloadBytesSucceeded; + private volatile long transferredBytesSucceeded; /** - * Time in milliseconds for the last successful copy operation from the source {@link org.apache.lucene.store.Directory} + * Time in milliseconds for the last successful transfer from the source {@link org.apache.lucene.store.Directory} */ - private volatile long lastDownloadTimestampMs; + private volatile long lastTransferTimestampMs; /** - * Provides moving average over the last N total size in bytes of files downloaded from the source {@link org.apache.lucene.store.Directory}. + * Provides moving average over the last N total size in bytes of files transferred from the source {@link org.apache.lucene.store.Directory}. * N is window size */ - private volatile MovingAverage downloadBytesMovingAverageReference; + private volatile MovingAverage transferredBytesMovingAverageReference; - private volatile long lastSuccessfulSegmentDownloadBytes; + private volatile long lastSuccessfulTransferInBytes; /** - * Provides moving average over the last N upload speed (in bytes/s) of segment files downloaded from the source {@link org.apache.lucene.store.Directory}. + * Provides moving average over the last N transfer speed (in bytes/s) of segment files transferred from the source {@link org.apache.lucene.store.Directory}. * N is window size */ - private volatile MovingAverage downloadBytesPerSecMovingAverageReference; + private volatile MovingAverage transferredBytesPerSecMovingAverageReference; private final int DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE = 20; - public long getDownloadBytesStarted() { - return downloadBytesStarted; + public long getTransferredBytesStarted() { + return transferredBytesStarted; } - public void addDownloadBytesStarted(long size) { - downloadBytesStarted += size; + public void addTransferredBytesStarted(long size) { + transferredBytesStarted += size; } - public long getDownloadBytesFailed() { - return downloadBytesFailed; + public long getTransferredBytesFailed() { + return transferredBytesFailed; } - public void addDownloadBytesFailed(long size) { - downloadBytesFailed += size; + public void addTransferredBytesFailed(long size) { + transferredBytesFailed += size; } - public long getDownloadBytesSucceeded() { - return downloadBytesSucceeded; + public long getTransferredBytesSucceeded() { + return transferredBytesSucceeded; } - public void addDownloadBytesSucceeded(long size) { - downloadBytesSucceeded += size; + public void addTransferredBytesSucceeded(long size, long startTimeInMs) { + transferredBytesSucceeded += size; + updateLastSuccessfulTransferSize(size); + long currentTimeInMs = System.currentTimeMillis(); + updateLastTransferTimestampMs(currentTimeInMs); + long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); + addTransferredBytesPerSec((size * 1_000L) / timeTakenInMS); } - public boolean isDownloadBytesPerSecAverageReady() { - return downloadBytesPerSecMovingAverageReference.isReady(); + public boolean isTransferredBytesPerSecAverageReady() { + return transferredBytesPerSecMovingAverageReference.isReady(); } - public double getDownloadBytesPerSecAverage() { - return downloadBytesPerSecMovingAverageReference.getAverage(); + public double getTransferredBytesPerSecAverage() { + return transferredBytesPerSecMovingAverageReference.getAverage(); } - public void addDownloadBytesPerSec(long bytesPerSec) { - this.downloadBytesPerSecMovingAverageReference.record(bytesPerSec); + // Visible for testing + public void addTransferredBytesPerSec(long bytesPerSec) { + this.transferredBytesPerSecMovingAverageReference.record(bytesPerSec); } - public boolean isDownloadBytesAverageReady() { - return downloadBytesMovingAverageReference.isReady(); + public boolean isTransferredBytesAverageReady() { + return transferredBytesMovingAverageReference.isReady(); } - public double getDownloadBytesAverage() { - return downloadBytesMovingAverageReference.getAverage(); + public double getTransferredBytesAverage() { + return transferredBytesMovingAverageReference.getAverage(); } - public void updateLastDownloadedSegmentSize(long size) { - lastSuccessfulSegmentDownloadBytes = size; - this.downloadBytesMovingAverageReference.record(size); + // Visible for testing + public void updateLastSuccessfulTransferSize(long size) { + lastSuccessfulTransferInBytes = size; + this.transferredBytesMovingAverageReference.record(size); } - public long getLastDownloadTimestampMs() { - return lastDownloadTimestampMs; + public long getLastTransferTimestampMs() { + return lastTransferTimestampMs; } - public void updateLastDownloadTimestampMs(long downloadTimestampInMs) { - this.lastDownloadTimestampMs = downloadTimestampInMs; + // Visible for testing + public void updateLastTransferTimestampMs(long downloadTimestampInMs) { + this.lastTransferTimestampMs = downloadTimestampInMs; } public DirectoryFileTransferTracker() { - downloadBytesMovingAverageReference = new MovingAverage(DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE); - downloadBytesPerSecMovingAverageReference = new MovingAverage(DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE); + transferredBytesMovingAverageReference = new MovingAverage(DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE); + transferredBytesPerSecMovingAverageReference = new MovingAverage(DIRECTORY_FILES_TRANSFER_DEFAULT_WINDOW_SIZE); } public DirectoryFileTransferTracker.Stats stats() { return new Stats( - downloadBytesStarted, - downloadBytesFailed, - downloadBytesSucceeded, - lastDownloadTimestampMs, - downloadBytesMovingAverageReference.getAverage(), - lastSuccessfulSegmentDownloadBytes, - downloadBytesPerSecMovingAverageReference.getAverage() + transferredBytesStarted, + transferredBytesFailed, + transferredBytesSucceeded, + lastTransferTimestampMs, + transferredBytesMovingAverageReference.getAverage(), + lastSuccessfulTransferInBytes, + transferredBytesPerSecMovingAverageReference.getAverage() ); } @@ -137,51 +145,51 @@ public DirectoryFileTransferTracker.Stats stats() { * @opensearch.internal */ public static class Stats implements Writeable { - public final long downloadBytesStarted; - public final long downloadBytesFailed; - public final long downloadBytesSucceeded; - public final long lastDownloadTimestampMs; - public final double downloadBytesMovingAverage; - public final long lastSuccessfulSegmentDownloadBytes; - public final double downloadBytesPerSecMovingAverage; + public final long transferredBytesStarted; + public final long transferredBytesFailed; + public final long transferredBytesSucceeded; + public final long lastTransferTimestampMs; + public final double transferredBytesMovingAverage; + public final long lastSuccessfulTransferInBytes; + public final double transferredBytesPerSecMovingAverage; public Stats( - long downloadBytesStarted, - long downloadBytesFailed, + long transferredBytesStarted, + long transferredBytesFailed, long downloadBytesSucceeded, - long lastDownloadTimestampMs, - double downloadBytesMovingAverage, - long lastSuccessfulSegmentDownloadBytes, - double downloadBytesPerSecMovingAverage + long lastTransferTimestampMs, + double transferredBytesMovingAverage, + long lastSuccessfulTransferInBytes, + double transferredBytesPerSecMovingAverage ) { - this.downloadBytesStarted = downloadBytesStarted; - this.downloadBytesFailed = downloadBytesFailed; - this.downloadBytesSucceeded = downloadBytesSucceeded; - this.lastDownloadTimestampMs = lastDownloadTimestampMs; - this.downloadBytesMovingAverage = downloadBytesMovingAverage; - this.lastSuccessfulSegmentDownloadBytes = lastSuccessfulSegmentDownloadBytes; - this.downloadBytesPerSecMovingAverage = downloadBytesPerSecMovingAverage; + this.transferredBytesStarted = transferredBytesStarted; + this.transferredBytesFailed = transferredBytesFailed; + this.transferredBytesSucceeded = downloadBytesSucceeded; + this.lastTransferTimestampMs = lastTransferTimestampMs; + this.transferredBytesMovingAverage = transferredBytesMovingAverage; + this.lastSuccessfulTransferInBytes = lastSuccessfulTransferInBytes; + this.transferredBytesPerSecMovingAverage = transferredBytesPerSecMovingAverage; } public Stats(StreamInput in) throws IOException { - this.downloadBytesStarted = in.readLong(); - this.downloadBytesFailed = in.readLong(); - this.downloadBytesSucceeded = in.readLong(); - this.lastDownloadTimestampMs = in.readLong(); - this.downloadBytesMovingAverage = in.readDouble(); - this.lastSuccessfulSegmentDownloadBytes = in.readLong(); - this.downloadBytesPerSecMovingAverage = in.readDouble(); + this.transferredBytesStarted = in.readLong(); + this.transferredBytesFailed = in.readLong(); + this.transferredBytesSucceeded = in.readLong(); + this.lastTransferTimestampMs = in.readLong(); + this.transferredBytesMovingAverage = in.readDouble(); + this.lastSuccessfulTransferInBytes = in.readLong(); + this.transferredBytesPerSecMovingAverage = in.readDouble(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeLong(downloadBytesStarted); - out.writeLong(downloadBytesFailed); - out.writeLong(downloadBytesSucceeded); - out.writeLong(lastDownloadTimestampMs); - out.writeDouble(downloadBytesMovingAverage); - out.writeLong(lastSuccessfulSegmentDownloadBytes); - out.writeDouble(downloadBytesPerSecMovingAverage); + out.writeLong(transferredBytesStarted); + out.writeLong(transferredBytesFailed); + out.writeLong(transferredBytesSucceeded); + out.writeLong(lastTransferTimestampMs); + out.writeDouble(transferredBytesMovingAverage); + out.writeLong(lastSuccessfulTransferInBytes); + out.writeDouble(transferredBytesPerSecMovingAverage); } } } diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 91b74b11093f3..a67b87f58110c 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -187,8 +187,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref // prevents segment files deletion until the PIT/Scroll expires or is discarded private final ReplicaFileTracker replicaFileTracker; - private final DirectoryFileTransferTracker directoryFileTransferTracker; - private final AbstractRefCounted refCounter = new AbstractRefCounted("store") { @Override protected void closeInternal() { @@ -206,12 +204,7 @@ public Store(ShardId shardId, IndexSettings indexSettings, Directory directory, final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING); logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval); ByteSizeCachingDirectory sizeCachingDir = new ByteSizeCachingDirectory(directory, refreshInterval); - this.directoryFileTransferTracker = new DirectoryFileTransferTracker(); - this.directory = new StoreDirectory( - sizeCachingDir, - directoryFileTransferTracker, - Loggers.getLogger("index.store.deletes", shardId) - ); + this.directory = new StoreDirectory(sizeCachingDir, Loggers.getLogger("index.store.deletes", shardId)); this.shardLock = shardLock; this.onClose = onClose; this.replicaFileTracker = indexSettings.isSegRepEnabled() ? new ReplicaFileTracker() : null; @@ -971,7 +964,7 @@ public void commitSegmentInfos(SegmentInfos latestSegmentInfos, long maxSeqNo, l } public DirectoryFileTransferTracker getDirectoryFileTransferTracker() { - return directoryFileTransferTracker; + return directory.getDirectoryFileTransferTracker(); } /** @@ -981,16 +974,13 @@ public DirectoryFileTransferTracker getDirectoryFileTransferTracker() { */ static final class StoreDirectory extends FilterDirectory { private final Logger deletesLogger; + public final DirectoryFileTransferTracker directoryFileTransferTracker; - StoreDirectory( - ByteSizeCachingDirectory delegateDirectory, - DirectoryFileTransferTracker directoryFileTransferTracker, - Logger deletesLogger - ) { + StoreDirectory(ByteSizeCachingDirectory delegateDirectory, Logger deletesLogger) { super(delegateDirectory); this.deletesLogger = deletesLogger; - this.directoryFileTransferTracker = directoryFileTransferTracker; + this.directoryFileTransferTracker = new DirectoryFileTransferTracker(); } /** Estimate the cumulative size of all files in this directory in bytes. */ @@ -1029,6 +1019,10 @@ public Set getPendingDeletions() throws IOException { return unwrap(this).getPendingDeletions(); } + public DirectoryFileTransferTracker getDirectoryFileTransferTracker() { + return directoryFileTransferTracker; + } + @Override public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { long fileSize = from.fileLength(src); @@ -1050,7 +1044,7 @@ public void copyFrom(Directory from, String src, String dest, IOContext context) * Updates the amount of bytes attempted for download */ private void beforeDownload(long fileSize) { - directoryFileTransferTracker.addDownloadBytesStarted(fileSize); + directoryFileTransferTracker.addTransferredBytesStarted(fileSize); } /** @@ -1061,19 +1055,14 @@ private void beforeDownload(long fileSize) { * - Download speed (in bytes/sec) */ private void afterDownload(long fileSize, long startTimeInMs) { - directoryFileTransferTracker.addDownloadBytesSucceeded(fileSize); - directoryFileTransferTracker.updateLastDownloadedSegmentSize(fileSize); - long currentTimeInMs = System.currentTimeMillis(); - directoryFileTransferTracker.updateLastDownloadTimestampMs(currentTimeInMs); - long timeTakenInMS = Math.max(1, currentTimeInMs - startTimeInMs); - directoryFileTransferTracker.addDownloadBytesPerSec((fileSize * 1_000L) / timeTakenInMS); + directoryFileTransferTracker.addTransferredBytesSucceeded(fileSize, startTimeInMs); } /** * Updates the amount of bytes failed in download */ private void downloadFailed(long fileSize) { - directoryFileTransferTracker.addDownloadBytesFailed(fileSize); + directoryFileTransferTracker.addTransferredBytesFailed(fileSize); } } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java index c30224da295e3..0c081ee238e2d 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTestHelper.java @@ -134,46 +134,46 @@ static void compareStatsResponse( Map segmentDownloads = ((Map) segment.get(RemoteStoreStats.SubFields.DOWNLOAD)); Map segmentUploads = ((Map) segment.get(RemoteStoreStats.SubFields.UPLOAD)); - if (statsTracker.directoryFileTransferTrackerStats.downloadBytesStarted != 0) { + if (statsTracker.directoryFileTransferTrackerStats.transferredBytesStarted != 0) { assertEquals( segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.LAST_SYNC_TIMESTAMP), - (int) statsTracker.directoryFileTransferTrackerStats.lastDownloadTimestampMs + (int) statsTracker.directoryFileTransferTrackerStats.lastTransferTimestampMs ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.STARTED ), - (int) statsTracker.directoryFileTransferTrackerStats.downloadBytesStarted + (int) statsTracker.directoryFileTransferTrackerStats.transferredBytesStarted ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.SUCCEEDED ), - (int) statsTracker.directoryFileTransferTrackerStats.downloadBytesSucceeded + (int) statsTracker.directoryFileTransferTrackerStats.transferredBytesSucceeded ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.TOTAL_DOWNLOADS_IN_BYTES)).get( RemoteStoreStats.SubFields.FAILED ), - (int) statsTracker.directoryFileTransferTrackerStats.downloadBytesFailed + (int) statsTracker.directoryFileTransferTrackerStats.transferredBytesFailed ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.LAST_SUCCESSFUL ), - (int) statsTracker.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes + (int) statsTracker.directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SIZE_IN_BYTES)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - statsTracker.directoryFileTransferTrackerStats.downloadBytesMovingAverage + statsTracker.directoryFileTransferTrackerStats.transferredBytesMovingAverage ); assertEquals( ((Map) segmentDownloads.get(RemoteStoreStats.DownloadStatsFields.DOWNLOAD_SPEED_IN_BYTES_PER_SEC)).get( RemoteStoreStats.SubFields.MOVING_AVG ), - statsTracker.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage + statsTracker.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage ); } else { assertTrue(segmentDownloads.isEmpty()); diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java index 044f21a3869af..3597a5350e1fb 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/RemoteStoreStatsTests.java @@ -103,11 +103,11 @@ public void testSerializationForPrimaryShard() throws Exception { assertEquals(stats.getStats().uploadBytesPerSecMovingAverage, deserializedStats.uploadBytesPerSecMovingAverage, 0); assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); - assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted); - assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.downloadBytesFailed); - assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded); - assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes); - assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.transferredBytesStarted); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.transferredBytesFailed); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.transferredBytesSucceeded); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes); + assertEquals(0, deserializedStats.directoryFileTransferTrackerStats.lastTransferTimestampMs); } } } @@ -132,33 +132,33 @@ public void testSerializationForReplicaShard() throws Exception { assertEquals(0, deserializedStats.consecutiveFailuresCount); assertEquals(0, deserializedStats.bytesLag); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesStarted, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted + stats.getStats().directoryFileTransferTrackerStats.transferredBytesStarted, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesStarted ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesFailed, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesFailed + stats.getStats().directoryFileTransferTrackerStats.transferredBytesFailed, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesFailed ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesSucceeded, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + stats.getStats().directoryFileTransferTrackerStats.transferredBytesSucceeded, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesSucceeded ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes, - deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes + stats.getStats().directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes, + deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.lastDownloadTimestampMs, - deserializedStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs + stats.getStats().directoryFileTransferTrackerStats.lastTransferTimestampMs, + deserializedStats.directoryFileTransferTrackerStats.lastTransferTimestampMs ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + stats.getStats().directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage, 0 ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesMovingAverage, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesMovingAverage, + stats.getStats().directoryFileTransferTrackerStats.transferredBytesMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesMovingAverage, 0 ); } @@ -188,33 +188,33 @@ public void testSerializationForRemoteStoreRestoredPrimaryShard() throws Excepti assertEquals(stats.getStats().uploadTimeMovingAverage, deserializedStats.uploadTimeMovingAverage, 0); assertEquals(stats.getStats().bytesLag, deserializedStats.bytesLag); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesStarted, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted + stats.getStats().directoryFileTransferTrackerStats.transferredBytesStarted, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesStarted ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesFailed, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesFailed + stats.getStats().directoryFileTransferTrackerStats.transferredBytesFailed, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesFailed ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesSucceeded, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + stats.getStats().directoryFileTransferTrackerStats.transferredBytesSucceeded, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesSucceeded ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes, - deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulSegmentDownloadBytes + stats.getStats().directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes, + deserializedStats.directoryFileTransferTrackerStats.lastSuccessfulTransferInBytes ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.lastDownloadTimestampMs, - deserializedStats.directoryFileTransferTrackerStats.lastDownloadTimestampMs + stats.getStats().directoryFileTransferTrackerStats.lastTransferTimestampMs, + deserializedStats.directoryFileTransferTrackerStats.lastTransferTimestampMs ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, + stats.getStats().directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage, 0 ); assertEquals( - stats.getStats().directoryFileTransferTrackerStats.downloadBytesMovingAverage, - deserializedStats.directoryFileTransferTrackerStats.downloadBytesMovingAverage, + stats.getStats().directoryFileTransferTrackerStats.transferredBytesMovingAverage, + deserializedStats.directoryFileTransferTrackerStats.transferredBytesMovingAverage, 0 ); } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java index 8fea8f98d940b..208cea111f2e1 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteSegmentTransferTrackerTests.java @@ -135,8 +135,8 @@ public void testLastDownloadTimestampMs() { pressureSettings.getUploadTimeMovingAverageWindowSize() ); long currentTimeInMs = System.currentTimeMillis(); - pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadTimestampMs(currentTimeInMs); - assertEquals(currentTimeInMs, pressureTracker.getDirectoryFileTransferTracker().getLastDownloadTimestampMs()); + pressureTracker.getDirectoryFileTransferTracker().updateLastTransferTimestampMs(currentTimeInMs); + assertEquals(currentTimeInMs, pressureTracker.getDirectoryFileTransferTracker().getLastTransferTimestampMs()); } public void testComputeSeqNoLagOnUpdate() { @@ -230,11 +230,11 @@ public void testAddDownloadBytesStarted() { pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesStarted(bytesToAdd); - assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesStarted()); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesStarted(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesStarted()); long moreBytesToAdd = randomLongBetween(1000, 10000); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesStarted(moreBytesToAdd); - assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesStarted()); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesStarted(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesStarted()); } public void testAddDownloadBytesFailed() { @@ -246,11 +246,11 @@ public void testAddDownloadBytesFailed() { pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesFailed(bytesToAdd); - assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesFailed()); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesFailed(bytesToAdd); + assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesFailed()); long moreBytesToAdd = randomLongBetween(1000, 10000); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesFailed(moreBytesToAdd); - assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesFailed()); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesFailed(moreBytesToAdd); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesFailed()); } public void testAddDownloadBytesSucceeded() { @@ -262,11 +262,11 @@ public void testAddDownloadBytesSucceeded() { pressureSettings.getUploadTimeMovingAverageWindowSize() ); long bytesToAdd = randomLongBetween(1000, 1000000); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesSucceeded(bytesToAdd); - assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesSucceeded()); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesSucceeded(bytesToAdd, System.currentTimeMillis()); + assertEquals(bytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesSucceeded()); long moreBytesToAdd = randomLongBetween(1000, 10000); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesSucceeded(moreBytesToAdd); - assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesSucceeded()); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesSucceeded(moreBytesToAdd, System.currentTimeMillis()); + assertEquals(bytesToAdd + moreBytesToAdd, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesSucceeded()); } public void testGetInflightUploadBytes() { @@ -498,24 +498,24 @@ public void testIsDownloadBytesAverageReady() { pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); - assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesAverageReady()); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isTransferredBytesAverageReady()); long sum = 0; for (int i = 1; i < 20; i++) { - pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadedSegmentSize(i); + pressureTracker.getDirectoryFileTransferTracker().updateLastSuccessfulTransferSize(i); sum += i; - assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesAverageReady()); - assertEquals((double) sum / i, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesAverage(), 0.0d); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isTransferredBytesAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesAverage(), 0.0d); } - pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadedSegmentSize(20); + pressureTracker.getDirectoryFileTransferTracker().updateLastSuccessfulTransferSize(20); sum += 20; - assertTrue(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesAverageReady()); - assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesAverage(), 0.0d); + assertTrue(pressureTracker.getDirectoryFileTransferTracker().isTransferredBytesAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesAverage(), 0.0d); - pressureTracker.getDirectoryFileTransferTracker().updateLastDownloadedSegmentSize(100); + pressureTracker.getDirectoryFileTransferTracker().updateLastSuccessfulTransferSize(100); sum = sum + 100 - 1; - assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesAverage(), 0.0d); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesAverage(), 0.0d); } public void testIsDownloadBytesPerSecAverageReady() { @@ -526,24 +526,24 @@ public void testIsDownloadBytesPerSecAverageReady() { pressureSettings.getUploadBytesPerSecMovingAverageWindowSize(), pressureSettings.getUploadTimeMovingAverageWindowSize() ); - assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesPerSecAverageReady()); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isTransferredBytesPerSecAverageReady()); long sum = 0; for (int i = 1; i < 20; i++) { - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(i); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesPerSec(i); sum += i; - assertFalse(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesPerSecAverageReady()); - assertEquals((double) sum / i, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesPerSecAverage(), 0.0d); + assertFalse(pressureTracker.getDirectoryFileTransferTracker().isTransferredBytesPerSecAverageReady()); + assertEquals((double) sum / i, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesPerSecAverage(), 0.0d); } - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(20); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesPerSec(20); sum += 20; - assertTrue(pressureTracker.getDirectoryFileTransferTracker().isDownloadBytesPerSecAverageReady()); - assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesPerSecAverage(), 0.0d); + assertTrue(pressureTracker.getDirectoryFileTransferTracker().isTransferredBytesPerSecAverageReady()); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesPerSecAverage(), 0.0d); - pressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(100); + pressureTracker.getDirectoryFileTransferTracker().addTransferredBytesPerSec(100); sum = sum + 100 - 1; - assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getDownloadBytesPerSecAverage(), 0.0d); + assertEquals((double) sum / 20, pressureTracker.getDirectoryFileTransferTracker().getTransferredBytesPerSecAverage(), 0.0d); } /** @@ -602,16 +602,16 @@ public void testStatsObjectCreationViaStream() throws IOException { assertEquals((int) deserializedStats.totalUploadsSucceeded, (int) pressureTrackerStats.totalUploadsSucceeded); assertEquals((int) deserializedStats.totalUploadsFailed, (int) pressureTrackerStats.totalUploadsFailed); assertEquals( - (int) deserializedStats.directoryFileTransferTrackerStats.downloadBytesStarted, - (int) pressureTrackerStats.directoryFileTransferTrackerStats.downloadBytesStarted + (int) deserializedStats.directoryFileTransferTrackerStats.transferredBytesStarted, + (int) pressureTrackerStats.directoryFileTransferTrackerStats.transferredBytesStarted ); assertEquals( - (int) deserializedStats.directoryFileTransferTrackerStats.downloadBytesSucceeded, - (int) pressureTrackerStats.directoryFileTransferTrackerStats.downloadBytesSucceeded + (int) deserializedStats.directoryFileTransferTrackerStats.transferredBytesSucceeded, + (int) pressureTrackerStats.directoryFileTransferTrackerStats.transferredBytesSucceeded ); assertEquals( - (int) deserializedStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage, - (int) pressureTrackerStats.directoryFileTransferTrackerStats.downloadBytesPerSecMovingAverage + (int) deserializedStats.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage, + (int) pressureTrackerStats.directoryFileTransferTrackerStats.transferredBytesPerSecMovingAverage ); } } @@ -630,9 +630,9 @@ private RemoteSegmentTransferTracker constructTracker() { segmentPressureTracker.addUploadBytes(99); segmentPressureTracker.updateRemoteRefreshTimeMs(System.nanoTime() / 1_000_000L + randomIntBetween(10, 100)); segmentPressureTracker.incrementRejectionCount(); - segmentPressureTracker.getDirectoryFileTransferTracker().addDownloadBytesStarted(10); - segmentPressureTracker.getDirectoryFileTransferTracker().addDownloadBytesSucceeded(10); - segmentPressureTracker.getDirectoryFileTransferTracker().addDownloadBytesPerSec(5); + segmentPressureTracker.getDirectoryFileTransferTracker().addTransferredBytesStarted(10); + segmentPressureTracker.getDirectoryFileTransferTracker().addTransferredBytesSucceeded(10, System.currentTimeMillis()); + segmentPressureTracker.getDirectoryFileTransferTracker().addTransferredBytesPerSec(5); return segmentPressureTracker; } }