Introduce includeShardsStats in the stats request to indicate that we only fetch a summary

docs/changelog/100466.yaml

@@ -0,0 +1,5 @@
+pr: 100466
+summary: "Introduce includeShardsStats in the stats request to indicate that we only fetch a summary"
+area: Stats
+type: enhancement
+issues: [99744]

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -136,7 +136,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_PACKAGE_LOADER_PLATFORM_ADDED = def(8_512_00_0);
     public static final TransportVersion PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME = def(8_513_00_0);
     public static final TransportVersion UNIVERSAL_PROFILING_LICENSE_ADDED = def(8_514_00_0);
-
+    public static final TransportVersion NEED_SHARDS_STATS_ADDED = def(8_515_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.stats;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.common.Strings;
@@ -33,6 +34,7 @@ public class NodesStatsRequest extends BaseNodesRequest<NodesStatsRequest> {
 
     private CommonStatsFlags indices = new CommonStatsFlags();
     private final Set<String> requestedMetrics = new HashSet<>();
+    private boolean includeShardsStats = true;
 
     public NodesStatsRequest() {
         super((String[]) null);
@@ -44,6 +46,11 @@ public NodesStatsRequest(StreamInput in) throws IOException {
         indices = new CommonStatsFlags(in);
         requestedMetrics.clear();
         requestedMetrics.addAll(in.readStringCollectionAsList());
+        if (in.getTransportVersion().onOrAfter(TransportVersions.NEED_SHARDS_STATS_ADDED)) {
+            includeShardsStats = in.readBoolean();
+        } else {
+            includeShardsStats = true;
+        }
     }
 
     /**
@@ -169,11 +176,22 @@ public String getDescription() {
         };
     }
 
+    public boolean includeShardsStats() {
+        return includeShardsStats;
+    }
+
+    public void setIncludeShardsStats(boolean includeShardsStats) {
+        this.includeShardsStats = includeShardsStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         indices.writeTo(out);
         out.writeStringCollection(requestedMetrics);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.NEED_SHARDS_STATS_ADDED)) {
+            out.writeBoolean(includeShardsStats);
+        }
     }
 
     /**

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java

@@ -83,6 +83,7 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest, Task task)
         Set<String> metrics = request.requestedMetrics();
         return nodeService.stats(
             request.indices(),
+            request.includeShardsStats(),
             NodesStatsRequest.Metric.OS.containedIn(metrics),
             NodesStatsRequest.Metric.PROCESS.containedIn(metrics),
             NodesStatsRequest.Metric.JVM.containedIn(metrics),

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -182,6 +182,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
         NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, false, true, false, false, false);
         NodeStats nodeStats = nodeService.stats(
             CommonStatsFlags.NONE,
+            false,
             true,
             true,
             true,

server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java

@@ -465,6 +465,7 @@ private DiskUsage getDiskUsage() {
                 false,
                 false,
                 false,
+                false,
                 true,
                 false,
                 false,

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -455,7 +455,7 @@ public boolean awaitClose(long timeout, TimeUnit timeUnit) throws InterruptedExc
         return closeLatch.await(timeout, timeUnit);
     }
 
-    public NodeIndicesStats stats(CommonStatsFlags flags) {
+    public NodeIndicesStats stats(CommonStatsFlags flags, boolean includeShardsStats) {
         CommonStats commonStats = new CommonStats(flags);
         // the cumulative statistics also account for shards that are no longer on this node, which is tracked by oldShardsStats
         for (Flag flag : flags.getFlags()) {
@@ -471,7 +471,7 @@ public NodeIndicesStats stats(CommonStatsFlags flags) {
             }
         }
 
-        return new NodeIndicesStats(commonStats, statsByIndex(this, flags), statsByShard(this, flags));
+        return new NodeIndicesStats(commonStats, statsByIndex(this, flags), statsByShard(this, flags), includeShardsStats);
     }
 
     static Map<Index, CommonStats> statsByIndex(final IndicesService indicesService, final CommonStatsFlags flags) {

server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java

@@ -59,6 +59,7 @@
 public class NodeIndicesStats implements Writeable, ChunkedToXContent {
 
     private static final TransportVersion VERSION_SUPPORTING_STATS_BY_INDEX = TransportVersions.V_8_5_0;
+    private static final Map<Index, List<IndexShardStats>> EMPTY_STATS_BY_SHARD = Map.of();
 
     private final CommonStats stats;
     private final Map<Index, List<IndexShardStats>> statsByShard;
@@ -86,8 +87,17 @@ public NodeIndicesStats(StreamInput in) throws IOException {
         }
     }
 
-    public NodeIndicesStats(CommonStats oldStats, Map<Index, CommonStats> statsByIndex, Map<Index, List<IndexShardStats>> statsByShard) {
-        this.statsByShard = Objects.requireNonNull(statsByShard);
+    public NodeIndicesStats(
+        CommonStats oldStats,
+        Map<Index, CommonStats> statsByIndex,
+        Map<Index, List<IndexShardStats>> statsByShard,
+        boolean includeShardsStats
+    ) {
+        if (includeShardsStats) {
+            this.statsByShard = Objects.requireNonNull(statsByShard);
+        } else {
+            this.statsByShard = EMPTY_STATS_BY_SHARD;
+        }
         this.statsByIndex = Objects.requireNonNull(statsByIndex);
 
         // make a total common stats from old ones and current ones
@@ -204,6 +214,11 @@ public DenseVectorStats getDenseVectorStats() {
         return stats.getDenseVectorStats();
     }
 
+    // for testing
+    Map<Index, List<IndexShardStats>> getStatsByShard() {
+        return statsByShard;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         stats.writeTo(out);

server/src/main/java/org/elasticsearch/node/NodeService.java

@@ -155,6 +155,7 @@ private Map<String, Integer> findComponentVersions() {
 
     public NodeStats stats(
         CommonStatsFlags indices,
+        boolean includeShardsStats,
         boolean os,
         boolean process,
         boolean jvm,
@@ -176,7 +177,7 @@ public NodeStats stats(
         return new NodeStats(
             transportService.getLocalNode(),
             System.currentTimeMillis(),
-            indices.anySet() ? indicesService.stats(indices) : null,
+            indices.anySet() ? indicesService.stats(indices, includeShardsStats) : null,
             os ? monitorService.osService().stats() : null,
             process ? monitorService.processService().stats() : null,
             jvm ? monitorService.jvmService().stats() : null,

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java

@@ -91,7 +91,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(nodesIds);
         nodesStatsRequest.timeout(request.param("timeout"));
         // level parameter validation
-        NodeStatsLevel.of(request, NodeStatsLevel.NODE);
+        final NodeStatsLevel nodeStatsLevel = NodeStatsLevel.of(request, NodeStatsLevel.NODE);
+        nodesStatsRequest.setIncludeShardsStats(nodeStatsLevel != NodeStatsLevel.NODE);
 
         if (metrics.size() == 1 && metrics.contains("_all")) {
             if (request.hasParam("index_metric")) {

server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -680,7 +680,7 @@ public static NodeStats createNodeStats() {
             statsByShard.put(indexTest, indexShardStats);
 
             CommonStats oldStats = new CommonStats(CommonStatsFlags.ALL);
-            nodeIndicesStats = new NodeIndicesStats(oldStats, statsByIndex, statsByShard);
+            nodeIndicesStats = new NodeIndicesStats(oldStats, statsByIndex, statsByShard, true);
         }
         OsStats osStats = null;
         if (frequently()) {

server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java

@@ -251,6 +251,7 @@ public void testNoDiskData() {
                 eq(false),
                 eq(false),
                 eq(false),
+                eq(false),
                 eq(true),
                 eq(false),
                 eq(false),
@@ -386,6 +387,7 @@ private void simulateDiskOutOfSpace() {
                 eq(false),
                 eq(false),
                 eq(false),
+                eq(false),
                 eq(true),
                 eq(false),
                 eq(false),
@@ -409,6 +411,7 @@ private void initializeIncreasedDiskSpaceUsage() {
                 eq(false),
                 eq(false),
                 eq(false),
+                eq(false),
                 eq(true),
                 eq(false),
                 eq(false),
@@ -432,6 +435,7 @@ private void simulateHealthDiskSpace() {
                 eq(false),
                 eq(false),
                 eq(false),
+                eq(false),
                 eq(true),
                 eq(false),
                 eq(false),

server/src/test/java/org/elasticsearch/indices/NodeIndicesStatsTests.java

@@ -8,18 +8,24 @@
 
 package org.elasticsearch.indices;
 
+import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.sameInstance;
 import static org.hamcrest.object.HasToString.hasToString;
 
 public class NodeIndicesStatsTests extends ESTestCase {
 
     public void testInvalidLevel() {
-        final NodeIndicesStats stats = new NodeIndicesStats(null, Collections.emptyMap(), Collections.emptyMap());
+        final NodeIndicesStats stats = new NodeIndicesStats(null, Collections.emptyMap(), Collections.emptyMap(), randomBoolean());
         final String level = randomAlphaOfLength(16);
         final ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("level", level));
         final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> stats.toXContentChunked(params));
@@ -29,4 +35,12 @@ public void testInvalidLevel() {
         );
     }
 
+    public void testIncludeShardsStatsFlag() {
+        final Map<Index, List<IndexShardStats>> statsByShards = new HashMap<>();
+        NodeIndicesStats stats = new NodeIndicesStats(null, Collections.emptyMap(), statsByShards, true);
+        assertThat(stats.getStatsByShard(), sameInstance(statsByShards));
+        stats = new NodeIndicesStats(null, Collections.emptyMap(), statsByShards, false);
+        assertThat(stats.getStatsByShard(), sameInstance(Map.of()));
+    }
+
 }

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -2460,6 +2460,7 @@ public void ensureEstimatedStats() {
                     false,
                     false,
                     false,
+                    false,
                     false
                 );
                 assertThat(

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/DataTiersUsageTransportActionTests.java

@@ -757,7 +757,7 @@ private List<NodeStats> buildNodeStats(ClusterState clusterState, long bytesPerS
                 IndexShardStats shardStats = new IndexShardStats(shardId, new ShardStats[] { shardStat });
                 indexStats.computeIfAbsent(shardId.getIndex(), k -> new ArrayList<>()).add(shardStats);
             }
-            NodeIndicesStats nodeIndexStats = new NodeIndicesStats(new CommonStats(), Collections.emptyMap(), indexStats);
+            NodeIndicesStats nodeIndexStats = new NodeIndicesStats(new CommonStats(), Collections.emptyMap(), indexStats, randomBoolean());
             nodeStatsList.add(mockNodeStats(node, nodeIndexStats));
         }
         return nodeStatsList;

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java

@@ -362,7 +362,7 @@ private static NodeStats mockNodeStats() {
         segmentsStats.addBitsetMemoryInBytes(++iota);
         indicesCommonStats.getSegments().add(segmentsStats);
 
-        final NodeIndicesStats indices = new NodeIndicesStats(indicesCommonStats, emptyMap(), emptyMap());
+        final NodeIndicesStats indices = new NodeIndicesStats(indicesCommonStats, emptyMap(), emptyMap(), randomBoolean());
 
         // Filesystem
         final FsInfo.DeviceStats ioStatsOne = new FsInfo.DeviceStats(