From b74186805901c28db9595ce10660e7037672f27f Mon Sep 17 00:00:00 2001 From: Pranshu Shukla Date: Wed, 10 Jul 2024 11:50:49 +0530 Subject: [PATCH] Modify optimisation approach without Cluster Settings Signed-off-by: Pranshu Shukla --- .../org/opensearch/nodestats/NodeStatsIT.java | 147 +++++++++++--- .../cluster/node/stats/NodesStatsRequest.java | 4 +- .../admin/indices/stats/CommonStatsFlags.java | 5 +- .../opensearch/indices/IndicesService.java | 1 - .../opensearch/indices/NodeIndicesStats.java | 101 ++++----- .../rest/action/cat/RestNodesAction.java | 3 +- .../cluster/node/stats/NodeStatsTests.java | 191 ++++++++++++++++++ 7 files changed, 360 insertions(+), 92 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java index 8f7bdc832a126..08f09a1421fd7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/nodestats/NodeStatsIT.java @@ -22,6 +22,11 @@ import org.opensearch.action.update.UpdateRequest; import org.opensearch.action.update.UpdateResponse; import org.opensearch.cluster.ClusterState; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; @@ -31,10 +36,13 @@ import org.opensearch.test.OpenSearchIntegTestCase.Scope; import org.hamcrest.MatcherAssert; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.ArrayList; import java.util.concurrent.atomic.AtomicLong; import static java.util.Collections.singletonMap; @@ -246,7 +254,13 @@ public void testNodeIndicesStatsDocStatusStatsCreateDeleteUpdate() { } } } - public void testNodeIndicesStatsOptimisedResponse() { + + /** + * Default behavior - without consideration of request level param on level, the NodeStatsRequest always + * returns ShardStats which is aggregated on the coordinator node when creating the XContent. + */ + public void testNodeIndicesStatsDefaultResponse() { + String testLevel = randomFrom("null", "node", "indices", "shards", "unknown"); internalCluster().startNode(); ensureGreen(); String indexName = "test1"; @@ -254,39 +268,126 @@ public void testNodeIndicesStatsOptimisedResponse() { refresh(); ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - NodesStatsResponse response = client().admin().cluster().prepareNodesStats().get(); + NodesStatsResponse response; + if (!testLevel.equals("null")) { + ArrayList level_arg = new ArrayList<>(); + level_arg.add(testLevel); + + CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); + commonStatsFlags.setLevels(level_arg.toArray(new String[0])); + response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get(); + } else { + response = client().admin().cluster().prepareNodesStats().get(); + } + response.getNodes().forEach(nodeStats -> { assertNotNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex())); - assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex())); + try { + // Without any param - default is level = nodes + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder = nodeStats.getIndices().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + Map xContentMap = xContentBuilderToMap(builder); + LinkedHashMap indicesStatsMap = (LinkedHashMap) xContentMap.get("indices"); + assertFalse(indicesStatsMap.containsKey("indices")); + assertFalse(indicesStatsMap.containsKey("shards")); + + // With param containing level as 'indices', the indices stats are returned + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder = nodeStats.getIndices() + .toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("level", "indices"))); + builder.endObject(); + + xContentMap = xContentBuilderToMap(builder); + indicesStatsMap = (LinkedHashMap) xContentMap.get("indices"); + assertTrue(indicesStatsMap.containsKey("indices")); + assertFalse(indicesStatsMap.containsKey("shards")); + + LinkedHashMap indexLevelStats = (LinkedHashMap) indicesStatsMap.get("indices"); + assertTrue(indexLevelStats.containsKey(indexName)); + + // With param containing level as 'shards', the shard stats are returned + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder = nodeStats.getIndices().toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("level", "shards"))); + builder.endObject(); + + xContentMap = xContentBuilderToMap(builder); + indicesStatsMap = (LinkedHashMap) xContentMap.get("indices"); + assertFalse(indicesStatsMap.containsKey("indices")); + assertTrue(indicesStatsMap.containsKey("shards")); + + LinkedHashMap shardLevelStats = (LinkedHashMap) indicesStatsMap.get("shards"); + assertTrue(shardLevelStats.containsKey(indexName)); + } catch (IOException e) { + throw new RuntimeException(e); + } }); + } + + /** + * Optimized behavior - to avoid unnecessary IO in the form of shard-stats when not required, we not honor the levels on the + * individual data nodes instead and pre-compute information as required. + */ + public void testNodeIndicesStatsOptimizedResponse() { + String testLevel = randomFrom("null", "node", "indices", "shards", "unknown"); + internalCluster().startNode(); + ensureGreen(); + String indexName = "test1"; + index(indexName, "type", "1", "f", "f"); + refresh(); + + NodesStatsResponse response; CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); commonStatsFlags.optimizeNodeIndicesStatsOnLevel(true); - response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get(); - response.getNodes().forEach(nodeStats -> { - assertNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex())); - assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex())); + if (!testLevel.equals("null")) { + ArrayList level_arg = new ArrayList<>(); + level_arg.add(testLevel); - }); - ArrayList level_arg = new ArrayList<>(); - level_arg.add("indices"); - commonStatsFlags.setLevels(level_arg.toArray(new String[0])); + commonStatsFlags.setLevels(level_arg.toArray(new String[0])); + } response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get(); - response.getNodes().forEach(nodeStats -> { - assertNotNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex())); - assertNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex())); - }); - level_arg.clear(); - level_arg.add("shards"); - commonStatsFlags.setLevels(level_arg.toArray(new String[0])); - response = client().admin().cluster().prepareNodesStats().setIndices(commonStatsFlags).get(); response.getNodes().forEach(nodeStats -> { - assertNotNull(nodeStats.getIndices().getShardStats(clusterState.metadata().index(indexName).getIndex())); - assertNull(nodeStats.getIndices().getIndexStats(clusterState.metadata().index(indexName).getIndex())); + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder = nodeStats.getIndices().toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("level", "shards"))); + builder.endObject(); + + Map xContentMap = xContentBuilderToMap(builder); + LinkedHashMap indicesStatsMap = (LinkedHashMap) xContentMap.get("indices"); + LinkedHashMap indicesStats = (LinkedHashMap) indicesStatsMap.get("indices"); + LinkedHashMap shardStats = (LinkedHashMap) indicesStatsMap.get("shards"); + + switch (testLevel) { + case "shards": + assertFalse(shardStats.isEmpty()); + assertFalse(indicesStats.isEmpty()); + break; + case "indices": + assertTrue(shardStats.isEmpty()); + assertFalse(indicesStats.isEmpty()); + break; + case "node": + case "null": + case "unknown": + assertTrue(shardStats.isEmpty()); + assertTrue(indicesStats.isEmpty()); + break; + } + } catch (IOException e) { + throw new RuntimeException(e); + } }); } - + private Map xContentBuilderToMap(XContentBuilder xContentBuilder) { + return XContentHelper.convertToMap(BytesReference.bytes(xContentBuilder), true, xContentBuilder.contentType()).v2(); + } private void assertDocStatusStats() { DocStatusStats docStatusStats = client().admin() diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 202a9cdf3cf4f..970c4340dd33b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -130,8 +130,8 @@ public NodesStatsRequest indices(boolean indices) { /** * Use Optimized Response filtered based on level */ - public NodesStatsRequest useOptimizedNodeIndicesStats(boolean useOptimizedNodeIndicesStats){ - if (this.indices!=null) { + public NodesStatsRequest useOptimizedNodeIndicesStats(boolean useOptimizedNodeIndicesStats) { + if (this.indices != null) { this.indices.optimizeNodeIndicesStatsOnLevel(true); } return this; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java index 5b69cc6eefb35..faccdcf02362c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java @@ -69,7 +69,6 @@ public class CommonStatsFlags implements Writeable, Cloneable { private String[] levels = new String[0]; private boolean optimizeNodeIndicesStatsOnLevel = false; - /** * @param flags flags to set. If no flags are supplied, default flags will be set. */ @@ -102,7 +101,7 @@ public CommonStatsFlags(StreamInput in) throws IOException { includeCaches = in.readEnumSet(CacheType.class); levels = in.readStringArray(); } - if (in.getVersion().onOrAfter(Version.V_2_15_0)) { + if (in.getVersion().onOrAfter(Version.V_2_16_0)) { optimizeNodeIndicesStatsOnLevel = in.readBoolean(); } } @@ -129,7 +128,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeEnumSet(includeCaches); out.writeStringArrayNullable(levels); } - if (out.getVersion().onOrAfter(Version.V_2_15_0)){ + if (out.getVersion().onOrAfter(Version.V_2_16_0)) { out.writeBoolean(optimizeNodeIndicesStatsOnLevel); } } diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 3ed524521dfec..ec4004bb6bb88 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -693,7 +693,6 @@ public NodeIndicesStats stats(CommonStatsFlags flags) { } } if (flags.optimizeNodeIndicesStatsOnLevel()) { - logger.info("Picked NodeIndicesStats"); return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats, flags.getLevels()); } return new NodeIndicesStats(commonStats, statsByShard(this, flags), searchRequestStats); diff --git a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java index a38e9289aca39..c3677745c716f 100644 --- a/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java +++ b/server/src/main/java/org/opensearch/indices/NodeIndicesStats.java @@ -32,8 +32,6 @@ package org.opensearch.indices; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexShardStats; @@ -78,13 +76,13 @@ */ @PublicApi(since = "1.0.0") public class NodeIndicesStats implements Writeable, ToXContentFragment { - private CommonStats stats; - private Map statsByIndex; - private Map> statsByShard; + protected CommonStats stats; + protected Map statsByIndex; + protected Map> statsByShard; public NodeIndicesStats(StreamInput in) throws IOException { stats = new CommonStats(in); - if (in.getVersion().onOrAfter(Version.V_2_15_0)) { + if (in.getVersion().onOrAfter(Version.V_2_16_0)) { // contains statsByIndex if (in.readBoolean()) { statsByIndex = new HashMap<>(); @@ -136,11 +134,17 @@ public NodeIndicesStats( } if (levels != null) { - if (Arrays.stream(levels).anyMatch(NodeIndicesStats.levels.indices::equals)) { - this.statsByIndex = createStatsByIndex(statsByShard); - } else if (Arrays.stream(levels).anyMatch(NodeIndicesStats.levels.shards::equals)) { - this.statsByShard = statsByShard; - } + Arrays.stream(levels).anyMatch(level -> { + switch (level) { + case Fields.INDICES: + this.statsByIndex = createStatsByIndex(statsByShard); + return true; + case Fields.SHARDS: + this.statsByShard = statsByShard; + return true; + } + return false; + }); } } @@ -250,7 +254,7 @@ public RecoveryStats getRecoveryStats() { public void writeTo(StreamOutput out) throws IOException { stats.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_2_15_0)) { + if (out.getVersion().onOrAfter(Version.V_2_16_0)) { out.writeBoolean(statsByIndex != null); if (statsByIndex != null) { writeStatsByIndex(out); @@ -300,29 +304,33 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.INDICES); stats.toXContent(builder, params); - if (levels.indices.equals(level)) { - builder.startObject(Fields.INDICES); - if (statsByIndex == null && statsByShard!=null) { + if (Fields.INDICES.equals(level)) { + if (statsByIndex == null && statsByShard != null) { statsByIndex = createStatsByIndex(statsByShard); } - for (Map.Entry entry : statsByIndex.entrySet()) { - builder.startObject(entry.getKey().getName()); - entry.getValue().toXContent(builder, params); - builder.endObject(); + builder.startObject(Fields.INDICES); + if (statsByIndex != null) { + for (Map.Entry entry : statsByIndex.entrySet()) { + builder.startObject(entry.getKey().getName()); + entry.getValue().toXContent(builder, params); + builder.endObject(); + } } builder.endObject(); - } else if (levels.shards.equals(level)) { + } else if (Fields.SHARDS.equals(level)) { builder.startObject("shards"); - for (Map.Entry> entry : statsByShard.entrySet()) { - builder.startArray(entry.getKey().getName()); - for (IndexShardStats indexShardStats : entry.getValue()) { - builder.startObject().startObject(String.valueOf(indexShardStats.getShardId().getId())); - for (ShardStats shardStats : indexShardStats.getShards()) { - shardStats.toXContent(builder, params); + if (statsByShard != null) { + for (Map.Entry> entry : statsByShard.entrySet()) { + builder.startArray(entry.getKey().getName()); + for (IndexShardStats indexShardStats : entry.getValue()) { + builder.startObject().startObject(String.valueOf(indexShardStats.getShardId().getId())); + for (ShardStats shardStats : indexShardStats.getShards()) { + shardStats.toXContent(builder, params); + } + builder.endObject().endObject(); } - builder.endObject().endObject(); + builder.endArray(); } - builder.endArray(); } builder.endObject(); } @@ -356,44 +364,13 @@ public List getShardStats(Index index) { } } - public CommonStats getIndexStats(Index index) { - if (statsByIndex == null) { - return null; - } else { - return statsByIndex.get(index); - } - } - /** * Fields used for parsing and toXContent * * @opensearch.internal */ - static final class Fields { - static final String INDICES = "indices"; - } - - /** - * Levels for the NodeIndicesStats - */ - public enum levels { - node("node"), - indices("indices"), - shards("shards"); - - private final String name; - - levels(String name) { - this.name = name; - } - - @Override - public String toString() { - return name; - } - - public boolean equals(String value) { - return this.name.equals(value); - } + public static final class Fields { + public static final String INDICES = "indices"; + public static final String SHARDS = "shards"; } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index 91894f2542536..cd5d76a29047d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -138,7 +138,8 @@ public void processResponse(final NodesInfoResponse nodesInfoResponse) { NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(); nodesStatsRequest.timeout(request.param("timeout")); nodesStatsRequest.clear() - .indices(true).useOptimizedNodeIndicesStats(true) + .indices(true) + .useOptimizedNodeIndicesStats(true) .addMetrics( NodesStatsRequest.Metric.JVM.metricName(), NodesStatsRequest.Metric.OS.metricName(), diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index f7bc96bdfe769..88db8efff41b6 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -32,13 +32,19 @@ package org.opensearch.action.admin.cluster.node.stats; +import org.opensearch.Version; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; +import org.opensearch.action.admin.indices.stats.IndexShardStats; +import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.action.search.SearchRequestStats; import org.opensearch.cluster.coordination.PendingClusterStateStats; import org.opensearch.cluster.coordination.PersistedStateStats; import org.opensearch.cluster.coordination.PublishClusterStateStats; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; +import org.opensearch.cluster.routing.TestShardRouting; import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.cluster.service.ClusterStateStats; @@ -53,6 +59,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.indices.breaker.AllCircuitBreakerStats; import org.opensearch.core.indices.breaker.CircuitBreakerStats; @@ -63,6 +70,10 @@ import org.opensearch.index.SegmentReplicationRejectionStats; import org.opensearch.index.remote.RemoteSegmentStats; import org.opensearch.index.remote.RemoteTranslogTransferTracker; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.shard.IndexingStats; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.StoreStats; import org.opensearch.index.translog.RemoteTranslogStats; import org.opensearch.indices.NodeIndicesStats; import org.opensearch.ingest.IngestStats; @@ -88,6 +99,7 @@ import org.opensearch.transport.TransportStats; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -1065,4 +1077,183 @@ private static RemoteTranslogTransferTracker.Stats getRandomRemoteTranslogTransf private OperationStats getPipelineStats(List pipelineStats, String id) { return pipelineStats.stream().filter(p1 -> p1.getPipelineId().equals(id)).findFirst().map(p2 -> p2.getStats()).orElse(null); } + + public static class MockNodeIndicesStats extends NodeIndicesStats { + + public MockNodeIndicesStats(StreamInput in) throws IOException { + super(in); + } + + public MockNodeIndicesStats( + CommonStats oldStats, + Map> statsByShard, + SearchRequestStats searchRequestStats + ) { + super(oldStats, statsByShard, searchRequestStats); + } + + public MockNodeIndicesStats( + CommonStats oldStats, + Map> statsByShard, + SearchRequestStats searchRequestStats, + String[] levels + ) { + super(oldStats, statsByShard, searchRequestStats, levels); + } + + public CommonStats getStats() { + return this.stats; + } + + public Map getStatsByIndex() { + return this.statsByIndex; + } + + public Map> getStatsByShard() { + return this.statsByShard; + } + } + + public void testNodeIndicesStatsSerializationWithOldESVersionNodes() throws IOException { + long numDocs = randomLongBetween(0, 10000); + long numDeletedDocs = randomLongBetween(0, 100); + CommonStats commonStats = new CommonStats(CommonStatsFlags.NONE); + + commonStats.docs = new DocsStats(numDocs, numDeletedDocs, 0); + commonStats.store = new StoreStats(100, 0L); + commonStats.indexing = new IndexingStats(); + + CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); + commonStatsFlags.clear(); + commonStatsFlags.set(CommonStatsFlags.Flag.Docs, true); + commonStatsFlags.set(CommonStatsFlags.Flag.Store, true); + commonStatsFlags.set(CommonStatsFlags.Flag.Indexing, true); + + Index newIndex = new Index("index", "_na_"); + + MockNodeIndicesStats mockNodeIndicesStats = generateMockNodeIndicesStats(commonStats, newIndex, commonStatsFlags); + + // To test out scenario when the incoming node stats response is from a node with an older ES Version. + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.setVersion(Version.V_2_13_0); + mockNodeIndicesStats.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + in.setVersion(Version.V_2_13_0); + MockNodeIndicesStats newNodeIndicesStats = new MockNodeIndicesStats(in); + + List incomingIndexStats = newNodeIndicesStats.getStatsByShard().get(newIndex); + incomingIndexStats.forEach(indexShardStats -> { + ShardStats shardStats = Arrays.stream(indexShardStats.getShards()).findFirst().get(); + DocsStats incomingDocStats = shardStats.getStats().docs; + + DocsStats hostDocStats = new DocsStats(numDocs, numDeletedDocs, 0); + assertEquals(incomingDocStats.getCount(), hostDocStats.getCount()); + assertEquals(incomingDocStats.getTotalSizeInBytes(), hostDocStats.getTotalSizeInBytes()); + assertEquals(incomingDocStats.getAverageSizeInBytes(), hostDocStats.getAverageSizeInBytes()); + assertEquals(incomingDocStats.getDeleted(), hostDocStats.getDeleted()); + }); + } + } + } + + public void testNodeIndicesStatsSerializationOnNewVersions() throws IOException { + long numDocs = randomLongBetween(0, 10000); + long numDeletedDocs = randomLongBetween(0, 100); + String levelParam = randomFrom("node", "indices", "shards"); + + CommonStats commonStats = new CommonStats(CommonStatsFlags.NONE); + + commonStats.docs = new DocsStats(numDocs, numDeletedDocs, 0); + commonStats.store = new StoreStats(100, 0L); + commonStats.indexing = new IndexingStats(); + + CommonStatsFlags commonStatsFlags = new CommonStatsFlags(); + commonStatsFlags.clear(); + commonStatsFlags.set(CommonStatsFlags.Flag.Docs, true); + commonStatsFlags.set(CommonStatsFlags.Flag.Store, true); + commonStatsFlags.set(CommonStatsFlags.Flag.Indexing, true); + commonStatsFlags.optimizeNodeIndicesStatsOnLevel(true); + + ArrayList level_arg = new ArrayList<>(); + level_arg.add(levelParam); + + commonStatsFlags.setLevels(level_arg.toArray(new String[0])); + + Index newIndex = new Index("index", "_na_"); + + MockNodeIndicesStats mockNodeIndicesStats = generateMockNodeIndicesStats(commonStats, newIndex, commonStatsFlags); + + // To test out scenario when the incoming node stats response is from a node with an older ES Version. + try (BytesStreamOutput out = new BytesStreamOutput()) { + mockNodeIndicesStats.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + MockNodeIndicesStats newNodeIndicesStats = new MockNodeIndicesStats(in); + switch (levelParam) { + case "node": + assertNull(newNodeIndicesStats.getStatsByIndex()); + assertNull(newNodeIndicesStats.getStatsByShard()); + break; + case "indices": + assertNull(newNodeIndicesStats.getStatsByShard()); + assertNotNull(newNodeIndicesStats.getStatsByIndex()); + break; + case "shards": + assertNull(newNodeIndicesStats.getStatsByIndex()); + assertNotNull(newNodeIndicesStats.getStatsByShard()); + break; + } + } + } + } + + public MockNodeIndicesStats generateMockNodeIndicesStats(CommonStats commonStats, Index index, CommonStatsFlags commonStatsFlags) { + DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + Map> statsByShard = new HashMap<>(); + List indexShardStatsList = new ArrayList<>(); + Index statsIndex = null; + for (int i = 0; i < 2; i++) { + ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)); + ShardRouting shardRouting = TestShardRouting.newShardRouting( + index.getName(), + i, + localNode.getId(), + randomBoolean(), + shardRoutingState + ); + + if (statsIndex == null) { + statsIndex = shardRouting.shardId().getIndex(); + } + + Path path = createTempDir().resolve("indices") + .resolve(shardRouting.shardId().getIndex().getUUID()) + .resolve(String.valueOf(shardRouting.shardId().id())); + + ShardStats shardStats = new ShardStats( + shardRouting, + new ShardPath(false, path, path, shardRouting.shardId()), + commonStats, + null, + null, + null + ); + IndexShardStats indexShardStats = new IndexShardStats(shardRouting.shardId(), new ShardStats[] { shardStats }); + indexShardStatsList.add(indexShardStats); + } + + statsByShard.put(statsIndex, indexShardStatsList); + + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + if (commonStatsFlags.optimizeNodeIndicesStatsOnLevel()) { + return new MockNodeIndicesStats( + new CommonStats(commonStatsFlags), + statsByShard, + new SearchRequestStats(clusterSettings), + commonStatsFlags.getLevels() + ); + } else { + return new MockNodeIndicesStats(new CommonStats(commonStatsFlags), statsByShard, new SearchRequestStats(clusterSettings)); + } + } }