Add Lucene segment-level fields stats

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -217,6 +217,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ENRICH_CACHE_STATS_SIZE_ADDED = def(8_707_00_0);
     public static final TransportVersion ENTERPRISE_GEOIP_DOWNLOADER = def(8_708_00_0);
     public static final TransportVersion NODES_STATS_ENUM_SET = def(8_709_00_0);
+    public static final TransportVersion SEGMENT_FIELDS_STATS = def(8_710_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -332,10 +332,11 @@ public NodeMappingStats getNodeMappingStats() {
         if (mapperService == null) {
             return null;
         }
-        long totalCount = mapperService().mappingLookup().getTotalMapperCount();
-        long totalEstimatedOverhead = totalCount * 1024L; // 1KiB estimated per mapping
-        NodeMappingStats indexNodeMappingStats = new NodeMappingStats(totalCount, totalEstimatedOverhead);
-        return indexNodeMappingStats;
+        long numFields = mapperService().mappingLookup().getTotalMapperCount();
+        long totalEstimatedOverhead = numFields * 1024L; // 1KiB estimated per mapping
+        // assumes all segments have the same mapping; otherwise, we need to acquire searchers to req
+        long numLeaves = shards.values().stream().mapToLong(s -> s.commitStats().getNumLeaves()).sum();
+        return new NodeMappingStats(numFields, totalEstimatedOverhead, numLeaves, numLeaves * numFields);
     }
 
     public Set<Integer> shardIds() {

server/src/main/java/org/elasticsearch/index/engine/CommitStats.java

@@ -8,6 +8,7 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.index.SegmentInfos;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -27,6 +28,7 @@ public final class CommitStats implements Writeable, ToXContentFragment {
     private final long generation;
     private final String id; // lucene commit id in base 64;
     private final int numDocs;
+    private final int numLeaves;
 
     public CommitStats(SegmentInfos segmentInfos) {
         // clone the map to protect against concurrent changes
@@ -35,13 +37,15 @@ public CommitStats(SegmentInfos segmentInfos) {
         generation = segmentInfos.getLastGeneration();
         id = Base64.getEncoder().encodeToString(segmentInfos.getId());
         numDocs = Lucene.getNumDocs(segmentInfos);
+        numLeaves = segmentInfos.size();
     }
 
     CommitStats(StreamInput in) throws IOException {
         userData = in.readImmutableMap(StreamInput::readString);
         generation = in.readLong();
         id = in.readOptionalString();
         numDocs = in.readInt();
+        numLeaves = in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_FIELDS_STATS) ? in.readVInt() : 0;
     }
 
     @Override
@@ -81,12 +85,19 @@ public int getNumDocs() {
         return numDocs;
     }
 
+    public int getNumLeaves() {
+        return numLeaves;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeMap(userData, StreamOutput::writeString);
         out.writeLong(generation);
         out.writeOptionalString(id);
         out.writeInt(numDocs);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_FIELDS_STATS)) {
+            out.writeVInt(numLeaves);
+        }
     }
 
     static final class Fields {

server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -30,10 +31,14 @@ private static final class Fields {
         static final String TOTAL_COUNT = "total_count";
         static final String TOTAL_ESTIMATED_OVERHEAD = "total_estimated_overhead";
         static final String TOTAL_ESTIMATED_OVERHEAD_IN_BYTES = "total_estimated_overhead_in_bytes";
+        static final String TOTAL_SEGMENT_FIELDS = "total_segment_fields";
+        static final String AVERAGE_FIELDS_PER_SEGMENT = "average_fields_per_segment";
     }
 
     private long totalCount;
     private long totalEstimatedOverhead;
+    private long totalSegments;
+    private long totalSegmentFields;
 
     public NodeMappingStats() {
 
@@ -42,17 +47,25 @@ public NodeMappingStats() {
     public NodeMappingStats(StreamInput in) throws IOException {
         totalCount = in.readVLong();
         totalEstimatedOverhead = in.readVLong();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_FIELDS_STATS)) {
+            totalSegments = in.readVLong();
+            totalSegmentFields = in.readVLong();
+        }
     }
 
-    public NodeMappingStats(long totalCount, long totalEstimatedOverhead) {
+    public NodeMappingStats(long totalCount, long totalEstimatedOverhead, long totalSegments, long totalSegmentFields) {
         this.totalCount = totalCount;
         this.totalEstimatedOverhead = totalEstimatedOverhead;
+        this.totalSegments = totalSegments;
+        this.totalSegmentFields = totalSegmentFields;
     }
 
     public void add(@Nullable NodeMappingStats other) {
         if (other == null) return;
         this.totalCount += other.totalCount;
         this.totalEstimatedOverhead += other.totalEstimatedOverhead;
+        this.totalSegments += other.totalSegments;
+        this.totalSegmentFields += other.totalSegmentFields;
     }
 
     public long getTotalCount() {
@@ -63,17 +76,31 @@ public ByteSizeValue getTotalEstimatedOverhead() {
         return ByteSizeValue.ofBytes(totalEstimatedOverhead);
     }
 
+    public long getTotalSegments() {
+        return totalSegments;
+    }
+
+    public long getTotalSegmentFields() {
+        return totalSegmentFields;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(totalCount);
         out.writeVLong(totalEstimatedOverhead);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.SEGMENT_FIELDS_STATS)) {
+            out.writeVLong(totalSegments);
+            out.writeVLong(totalSegmentFields);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.MAPPINGS);
         builder.field(Fields.TOTAL_COUNT, getTotalCount());
         builder.humanReadableField(Fields.TOTAL_ESTIMATED_OVERHEAD_IN_BYTES, Fields.TOTAL_ESTIMATED_OVERHEAD, getTotalEstimatedOverhead());
+        builder.field(Fields.TOTAL_SEGMENT_FIELDS, totalSegments);
+        builder.field(Fields.AVERAGE_FIELDS_PER_SEGMENT, totalSegments == 0 ? 0 : totalSegmentFields / totalSegments);
         builder.endObject();
         return builder;
     }
@@ -83,11 +110,14 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         NodeMappingStats that = (NodeMappingStats) o;
-        return totalCount == that.totalCount && totalEstimatedOverhead == that.totalEstimatedOverhead;
+        return totalCount == that.totalCount
+            && totalEstimatedOverhead == that.totalEstimatedOverhead
+            && totalSegments == that.totalSegments
+            && totalSegmentFields == that.totalSegmentFields;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(totalCount, totalEstimatedOverhead);
+        return Objects.hash(totalCount, totalEstimatedOverhead, totalSegments, totalSegmentFields);
     }
 }

server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -561,7 +561,12 @@ private static int expectedChunks(@Nullable NodeIndicesStats nodeIndicesStats, N
 
     private static CommonStats createIndexLevelCommonStats() {
         CommonStats stats = new CommonStats(new CommonStatsFlags().clear().set(CommonStatsFlags.Flag.Mappings, true));
-        stats.nodeMappings = new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong());
+        stats.nodeMappings = new NodeMappingStats(
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            randomNonNegativeLong()
+        );
         return stats;
     }
 

server/src/test/java/org/elasticsearch/index/mapper/NodeMappingStatsTests.java

@@ -7,52 +7,62 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-public class NodeMappingStatsTests extends ESTestCase {
-
-    public void testSerialize() throws IOException {
-        NodeMappingStats stats = randomNodeMappingStats();
-        BytesStreamOutput out = new BytesStreamOutput();
-        stats.writeTo(out);
-        StreamInput input = out.bytes().streamInput();
-        NodeMappingStats read = new NodeMappingStats(input);
-        assertEquals(-1, input.read());
-        assertEquals(stats.getTotalCount(), read.getTotalCount());
-        assertEquals(stats.getTotalEstimatedOverhead(), read.getTotalEstimatedOverhead());
-    }
+public class NodeMappingStatsTests extends AbstractWireSerializingTestCase<NodeMappingStats> {
 
-    public void testEqualityAndHashCode() {
-        NodeMappingStats stats = randomNodeMappingStats();
-        assertEquals(stats, stats);
-        assertEquals(stats.hashCode(), stats.hashCode());
+    @Override
+    protected Writeable.Reader<NodeMappingStats> instanceReader() {
+        return NodeMappingStats::new;
+    }
 
-        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L);
-        NodeMappingStats stats2 = new NodeMappingStats(3L, 5L);
-        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L);
+    @Override
+    protected NodeMappingStats createTestInstance() {
+        return new NodeMappingStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+    }
 
-        assertNotEquals(stats1, stats2);
-        assertNotEquals(stats1, stats3);
-        assertEquals(stats2, stats3);
+    @Override
+    protected NodeMappingStats mutateInstance(NodeMappingStats in) throws IOException {
+        return switch (between(0, 3)) {
+            case 0 -> new NodeMappingStats(
+                randomValueOtherThan(in.getTotalCount(), ESTestCase::randomNonNegativeLong),
+                in.getTotalEstimatedOverhead().getBytes(),
+                in.getTotalSegments(),
+                in.getTotalSegmentFields()
+            );
+            case 1 -> new NodeMappingStats(
+                in.getTotalCount(),
+                randomValueOtherThan(in.getTotalCount(), ESTestCase::randomNonNegativeLong),
+                in.getTotalSegments(),
+                in.getTotalSegmentFields()
+            );
+            case 2 -> new NodeMappingStats(
+                in.getTotalCount(),
+                in.getTotalEstimatedOverhead().getBytes(),
+                randomValueOtherThan(in.getTotalSegments(), ESTestCase::randomNonNegativeLong),
+                in.getTotalSegmentFields()
+            );
+            case 3 -> new NodeMappingStats(
+                in.getTotalCount(),
+                in.getTotalEstimatedOverhead().getBytes(),
+                in.getTotalSegments(),
+                randomValueOtherThan(in.getTotalSegmentFields(), ESTestCase::randomNonNegativeLong)
+            );
+            default -> throw new AssertionError("invalid option");
+        };
     }
 
     public void testAdd() {
-        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L);
-        NodeMappingStats stats2 = new NodeMappingStats(2L, 3L);
-        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L);
+        NodeMappingStats stats1 = new NodeMappingStats(1L, 2L, 4L, 6L);
+        NodeMappingStats stats2 = new NodeMappingStats(2L, 3L, 10L, 20L);
+        NodeMappingStats stats3 = new NodeMappingStats(3L, 5L, 14L, 26L);
 
         stats1.add(stats2);
         assertEquals(stats1, stats3);
         assertEquals(stats1.hashCode(), stats3.hashCode());
     }
-
-    private static NodeMappingStats randomNodeMappingStats() {
-        long totalCount = randomIntBetween(1, 100);
-        long estimatedOverhead = totalCount * 1024;
-        return new NodeMappingStats(totalCount, estimatedOverhead);
-    }
 }