From fe4c3e745c5d7ff4fd4aa3a2f1d78bac45cd55bc Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 15 Apr 2021 10:15:26 +0200 Subject: [PATCH] Enhanced segment files sizes information in Nodes Stats/Indices Stats APIs (#71643) Since #16661 it is possible to know the total sizes for some Lucene segment files by using the Node Stats or Indices Stats API with the include_segment_file_sizes parameter, and the list of file extensions has been extended in #71416. This commit adds a bit more information about file sizes like the number of files (count), the min, max and average file sizes in bytes that share the same extension. Here is a sample: "cfs" : { "description" : "Compound Files", "size_in_bytes" : 2260, "min_size_in_bytes" : 2260, "max_size_in_bytes" : 2260, "average_size_in_bytes" : 2260, "count" : 1 } This commit also simplifies how compound file sizes were computed: before compound segment files were extracted and sizes aggregated with regular non-compound files sizes (which can be confusing and out of the scope of the original issue #6728), now CFS/CFE files appears as distinct files. These new information are provided to give a better view of the segment files and are useful in many cases, specially with frozen searchable snapshots whose segment stats can now be introspected thanks to the include_unloaded_segments parameter. --- .../test/indices.stats/30_segments.yml | 39 ++++ .../indices/stats/IndexStatsIT.java | 16 +- .../elasticsearch/index/engine/Engine.java | 110 ++++-------- .../index/engine/NoOpEngine.java | 2 +- .../index/engine/SegmentsStats.java | 170 ++++++++++++++---- .../index/engine/InternalEngineTests.java | 26 ++- .../index/engine/NoOpEngineTests.java | 6 +- .../index/engine/FrozenEngine.java | 2 +- .../rest-api-spec/test/indices_stats.yml | 156 ++++++++++++++++ 9 files changed, 397 insertions(+), 130 deletions(-) create mode 100644 x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/indices_stats.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/30_segments.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/30_segments.yml index e4e820ff2b8bb..9dc770d06574d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/30_segments.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/30_segments.yml @@ -61,3 +61,42 @@ setup: forbid_closed_indices: false - match: { indices.test.primaries.segments.count: $num_segments_after_flush } + + +--- +"Indices Stats API with extended files stats": + + - skip: + version: " - 7.99.99" + reason: segment files stats extended in 8.0.0 + + - do: + index: + index: test + id: 1 + body: { "foo": "bar" } + + - do: + indices.flush: + index: test + + - do: + indices.stats: + metric: [ segments ] + include_segment_file_sizes: true + + - is_true: _all.total.segments.file_sizes + - is_true: _all.total.segments.file_sizes.si + - gt: { _all.total.segments.file_sizes.si.count: 0 } + - gt: { _all.total.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.average_size_in_bytes: 0 } + + - is_true: indices.test.primaries.segments.file_sizes + - is_true: indices.test.primaries.segments.file_sizes.si + - gt: { indices.test.primaries.segments.file_sizes.si.count: 0 } + - gt: { indices.test.primaries.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { indices.test.primaries.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { indices.test.primaries.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { indices.test.primaries.segments.file_sizes.si.average_size_in_bytes: 0 } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index b1ed676c6accb..8f1a00c0f679b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.indices.stats; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -44,6 +45,7 @@ import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.cache.query.QueryCacheStats; +import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.IndexShard; @@ -607,11 +609,23 @@ public void testSegmentsStats() { client().admin().indices().prepareFlush().get(); client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet(); client().admin().indices().prepareRefresh().get(); - stats = client().admin().indices().prepareStats().setSegments(true).get(); + + final boolean includeSegmentFileSizes = randomBoolean(); + stats = client().admin().indices().prepareStats().setSegments(true).setIncludeSegmentFileSizes(includeSegmentFileSizes).get(); assertThat(stats.getTotal().getSegments(), notNullValue()); assertThat(stats.getTotal().getSegments().getCount(), equalTo((long) test1.totalNumShards)); assertThat(stats.getTotal().getSegments().getMemoryInBytes(), greaterThan(0L)); + if (includeSegmentFileSizes) { + assertThat(stats.getTotal().getSegments().getFiles().size(), greaterThan(0)); + for (ObjectObjectCursor cursor : stats.getTotal().getSegments().getFiles()) { + assertThat(cursor.value.getExt(), notNullValue()); + assertThat(cursor.value.getTotal(), greaterThan(0L)); + assertThat(cursor.value.getCount(), greaterThan(0L)); + assertThat(cursor.value.getMin(), greaterThan(0L)); + assertThat(cursor.value.getMax(), greaterThan(0L)); + } + } } public void testAllFlags() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 7d662db643b67..be5c57de7fcda 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -25,8 +25,6 @@ import org.apache.lucene.search.ReferenceManager; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; import org.apache.lucene.util.SetOnce; @@ -66,10 +64,8 @@ import org.elasticsearch.search.suggest.completion.CompletionStats; import java.io.Closeable; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.NoSuchFileException; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; @@ -135,7 +131,7 @@ protected Engine(EngineConfig engineConfig) { this.store = engineConfig.getStore(); // we use the engine class directly here to make sure all subclasses have the same logger name this.logger = Loggers.getLogger(Engine.class, - engineConfig.getShardId()); + engineConfig.getShardId()); this.eventListener = engineConfig.getEventListener(); } @@ -178,7 +174,7 @@ public DocsStats docStats() { // when indexing but not refreshing in general. Yet, if a refresh happens the internal searcher is refresh as well so we are // safe here. try (Searcher searcher = acquireSearcher("docStats", SearcherScope.INTERNAL)) { - return docsStats(searcher.getIndexReader()); + return docsStats(searcher.getIndexReader()); } } @@ -285,12 +281,14 @@ boolean throttleLockIsHeldByCurrentThread() { // to be used in assertions and te /** * Returns the true iff this engine is currently under index throttling. + * * @see #getIndexThrottleTimeInMillis() */ public abstract boolean isThrottled(); /** * Trims translog for terms below belowTerm and seq# above aboveSeqNo + * * @see Translog#trimOperations(long, long) */ public abstract void trimOperationsFromTranslog(long belowTerm, long aboveSeqNo) throws EngineException; @@ -785,86 +783,38 @@ protected void fillSegmentStats(SegmentReader segmentReader, boolean includeSegm stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); - if (includeSegmentFileSizes) { - // TODO: consider moving this to StoreStats - stats.addFileSizes(getSegmentFileSizes(segmentReader)); + stats.addFiles(getSegmentFileSizes(segmentReader)); } } - private ImmutableOpenMap getSegmentFileSizes(SegmentReader segmentReader) { - Directory directory = null; - SegmentCommitInfo segmentCommitInfo = segmentReader.getSegmentInfo(); - boolean useCompoundFile = segmentCommitInfo.info.getUseCompoundFile(); - if (useCompoundFile) { - try { - directory = engineConfig.getCodec().compoundFormat().getCompoundReader(segmentReader.directory(), - segmentCommitInfo.info, IOContext.READ); - } catch (IOException e) { - logger.warn(() -> new ParameterizedMessage("Error when opening compound reader for Directory [{}] and " + - "SegmentCommitInfo [{}]", segmentReader.directory(), segmentCommitInfo), e); - - return ImmutableOpenMap.of(); - } - } else { - directory = segmentReader.directory(); - } - - assert directory != null; - - String[] files; - if (useCompoundFile) { - try { - files = directory.listAll(); - } catch (IOException e) { - final Directory finalDirectory = directory; - logger.warn(() -> - new ParameterizedMessage("Couldn't list Compound Reader Directory [{}]", finalDirectory), e); - return ImmutableOpenMap.of(); - } - } else { - try { - files = segmentReader.getSegmentInfo().files().toArray(new String[]{}); - } catch (IOException e) { - logger.warn(() -> - new ParameterizedMessage("Couldn't list Directory from SegmentReader [{}] and SegmentInfo [{}]", - segmentReader, segmentReader.getSegmentInfo()), e); - return ImmutableOpenMap.of(); - } - } - - ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(); - for (String file : files) { - String extension = IndexFileNames.getExtension(file); - long length = 0L; - try { - length = directory.fileLength(file); - } catch (NoSuchFileException | FileNotFoundException e) { - final Directory finalDirectory = directory; - logger.warn(() -> new ParameterizedMessage("Tried to query fileLength but file is gone [{}] [{}]", - finalDirectory, file), e); - } catch (IOException e) { - final Directory finalDirectory = directory; - logger.warn(() -> new ParameterizedMessage("Error when trying to query fileLength [{}] [{}]", - finalDirectory, file), e); - } - if (length == 0L) { - continue; - } - map.put(extension, length); - } - - if (useCompoundFile) { - try { - directory.close(); - } catch (IOException e) { - final Directory finalDirectory = directory; - logger.warn(() -> new ParameterizedMessage("Error when closing compound reader on Directory [{}]", - finalDirectory), e); + private ImmutableOpenMap getSegmentFileSizes(SegmentReader segmentReader) { + try { + final ImmutableOpenMap.Builder files = ImmutableOpenMap.builder(); + final SegmentCommitInfo segmentCommitInfo = segmentReader.getSegmentInfo(); + for (String fileName : segmentCommitInfo.files()) { + String fileExtension = IndexFileNames.getExtension(fileName); + if (fileExtension != null) { + try { + long fileLength = segmentReader.directory().fileLength(fileName); + files.put(fileExtension, new SegmentsStats.FileStats(fileExtension, fileLength, 1L, fileLength, fileLength)); + } catch (IOException ioe) { + logger.warn(() -> + new ParameterizedMessage("Error when retrieving file length for [{}]", fileName), ioe); + } catch (AlreadyClosedException ace) { + logger.warn(() -> + new ParameterizedMessage("Error when retrieving file length for [{}], directory is closed", fileName), ace); + return ImmutableOpenMap.of(); + } + } } + return files.build(); + } catch (IOException e) { + logger.warn(() -> + new ParameterizedMessage("Error when listing files for segment reader [{}] and segment info [{}]", + segmentReader, segmentReader.getSegmentInfo()), e); + return ImmutableOpenMap.of(); } - - return map.build(); } protected void writerSegmentStats(SegmentsStats stats) { diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java index 7a9692aa7989b..1b09be36e6001 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java @@ -110,7 +110,7 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl final SegmentsStats stats = new SegmentsStats(); stats.add(this.segmentsStats); if (includeSegmentFileSizes == false) { - stats.clearFileSizes(); + stats.clearFiles(); } return stats; } else { diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java index 0e5cad2daf5b1..71cba7e92fb43 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java +++ b/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java @@ -8,7 +8,10 @@ package org.elasticsearch.index.engine; +import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.Version; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,9 +37,10 @@ public class SegmentsStats implements Writeable, ToXContentFragment { private long versionMapMemoryInBytes; private long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE; private long bitsetMemoryInBytes; - private ImmutableOpenMap fileSizes = ImmutableOpenMap.of(); + private ImmutableOpenMap files = ImmutableOpenMap.of(); - public SegmentsStats() {} + public SegmentsStats() { + } public SegmentsStats(StreamInput in) throws IOException { count = in.readVLong(); @@ -52,14 +56,13 @@ public SegmentsStats(StreamInput in) throws IOException { bitsetMemoryInBytes = in.readLong(); maxUnsafeAutoIdTimestamp = in.readLong(); - int size = in.readVInt(); - ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(size); + final int size = in.readVInt(); + final ImmutableOpenMap.Builder files = ImmutableOpenMap.builder(size); for (int i = 0; i < size; i++) { - String key = in.readString(); - Long value = in.readLong(); - map.put(key, value); + FileStats file = new FileStats(in); + files.put(file.getExt(), file); } - fileSizes = map.build(); + this.files = files.build(); } public void add(long count, long memoryInBytes) { @@ -107,19 +110,18 @@ public void addBitsetMemoryInBytes(long bitsetMemoryInBytes) { this.bitsetMemoryInBytes += bitsetMemoryInBytes; } - public void addFileSizes(ImmutableOpenMap fileSizes) { - ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(this.fileSizes); - - for (ObjectObjectCursor entry : fileSizes) { - if (map.containsKey(entry.key)) { - Long oldValue = map.get(entry.key); - map.put(entry.key, oldValue + entry.value); + public void addFiles(ImmutableOpenMap files) { + final ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(this.files); + for (ObjectObjectCursor entry : files) { + final String extension = entry.key; + if (map.containsKey(extension)) { + FileStats previous = map.get(extension); + map.put(extension, FileStats.merge(previous, entry.value)); } else { - map.put(entry.key, entry.value); + map.put(extension, entry.value); } } - - this.fileSizes = map.build(); + this.files = map.build(); } public void add(SegmentsStats mergeStats) { @@ -137,7 +139,7 @@ public void add(SegmentsStats mergeStats) { addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes); addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes); addBitsetMemoryInBytes(mergeStats.bitsetMemoryInBytes); - addFileSizes(mergeStats.fileSizes); + addFiles(mergeStats.files); } /** @@ -257,8 +259,8 @@ public ByteSizeValue getBitsetMemory() { return new ByteSizeValue(bitsetMemoryInBytes); } - public ImmutableOpenMap getFileSizes() { - return fileSizes; + public ImmutableOpenMap getFiles() { + return files; } /** @@ -285,12 +287,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.humanReadableField(Fields.FIXED_BIT_SET_MEMORY_IN_BYTES, Fields.FIXED_BIT_SET, getBitsetMemory()); builder.field(Fields.MAX_UNSAFE_AUTO_ID_TIMESTAMP, maxUnsafeAutoIdTimestamp); builder.startObject(Fields.FILE_SIZES); - for (ObjectObjectCursor entry : fileSizes) { - builder.startObject(entry.key); - builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(entry.value)); - LuceneFilesExtensions extension = LuceneFilesExtensions.fromExtension(entry.key); - builder.field(Fields.DESCRIPTION, extension != null ? extension.getDescription() : "Others"); - builder.endObject(); + for (ObjectObjectCursor entry : files) { + entry.value.toXContent(builder, params); } builder.endObject(); builder.endObject(); @@ -322,9 +320,6 @@ static final class Fields { static final String FIXED_BIT_SET = "fixed_bit_set"; static final String FIXED_BIT_SET_MEMORY_IN_BYTES = "fixed_bit_set_memory_in_bytes"; static final String FILE_SIZES = "file_sizes"; - static final String SIZE = "size"; - static final String SIZE_IN_BYTES = "size_in_bytes"; - static final String DESCRIPTION = "description"; } @Override @@ -342,14 +337,117 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(bitsetMemoryInBytes); out.writeLong(maxUnsafeAutoIdTimestamp); - out.writeVInt(fileSizes.size()); - for (ObjectObjectCursor entry : fileSizes) { - out.writeString(entry.key); - out.writeLong(entry.value); + out.writeVInt(files.size()); + for (ObjectCursor file : files.values()) { + file.value.writeTo(out); } } - public void clearFileSizes() { - fileSizes = ImmutableOpenMap.of(); + public void clearFiles() { + files = ImmutableOpenMap.of(); + } + + public static class FileStats implements Writeable, ToXContentFragment { + + private final String ext; + private final long total; + private final long count; + private final long min; + private final long max; + + FileStats(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + this.ext = in.readString(); + this.total = in.readVLong(); + this.count = in.readVLong(); + this.min = in.readVLong(); + this.max = in.readVLong(); + } else { + this.ext = in.readString(); + this.total = in.readLong(); + this.count = 0L; + this.min = 0L; + this.max = 0L; + } + } + + public FileStats(String ext, long total, long count, long min, long max) { + this.ext = ext; + this.total = total; + this.count = count; + this.min = min; + this.max = max; + } + + public String getExt() { + return ext; + } + + public long getCount() { + return count; + } + + public long getTotal() { + return total; + } + + public long getMin() { + return min; + } + + public long getMax() { + return max; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeString(ext); + out.writeVLong(total); + out.writeVLong(count); + out.writeVLong(min); + out.writeVLong(max); + } else { + out.writeString(ext); + out.writeLong(total); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + final long roundedAverage = count > 0L ? Math.round((double) total / (double) count) : 0L; + final LuceneFilesExtensions extension = LuceneFilesExtensions.fromExtension(ext); + final String name = extension != null ? extension.getExtension() : "others"; + final String desc = extension != null ? extension.getDescription() : "Others"; + builder.startObject(name); + { + builder.field("description", desc); + builder.humanReadableField("size_in_bytes", "size", ByteSizeValue.ofBytes(total)); + builder.humanReadableField("min_size_in_bytes", "min_size", ByteSizeValue.ofBytes(min)); + builder.humanReadableField("max_size_in_bytes", "max_size", ByteSizeValue.ofBytes(max)); + builder.humanReadableField("average_size_in_bytes", "average_size", ByteSizeValue.ofBytes(roundedAverage)); + builder.field("count", count); + } + builder.endObject(); + return builder; + } + + public static FileStats merge(FileStats o1, FileStats o2) { + assert o1 != null && o1.ext != null : o1; + assert o2 != null && o2.ext != null : o2; + assert o1.ext.equals(o2.ext) : o1 + " vs " + o2; + return new FileStats( + o1.ext, + Math.addExact(o1.total, o2.total), + Math.addExact(o1.count, o2.count), + Math.min(o1.min, o2.min), + Math.max(o1.max, o2.max) + ); + } + + @Override + public String toString() { + return Strings.toString(this); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 2b34d232ec507..f3d4a809e835f 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.engine; +import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.logging.log4j.Level; @@ -185,7 +186,6 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasItem; @@ -388,23 +388,33 @@ public void testSegmentsWithIndexSort() throws Exception { public void testSegmentsStatsIncludingFileSizes() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { - assertThat(engine.segmentsStats(true, false).getFileSizes().size(), equalTo(0)); + assertThat(engine.segmentsStats(true, false).getFiles().size(), equalTo(0)); ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null); engine.index(indexForDoc(doc)); engine.refresh("test"); - SegmentsStats stats = engine.segmentsStats(true, false); - assertThat(stats.getFileSizes().size(), greaterThan(0)); - assertThat(() -> stats.getFileSizes().valuesIt(), everyItem(greaterThan(0L))); - - ObjectObjectCursor firstEntry = stats.getFileSizes().iterator().next(); + final SegmentsStats stats1 = engine.segmentsStats(true, false); + assertThat(stats1.getFiles().size(), greaterThan(0)); + for (ObjectObjectCursor fileStats : stats1.getFiles()) { + assertThat(fileStats.value.getTotal(), greaterThan(0L)); + assertThat(fileStats.value.getCount(), greaterThan(0L)); + assertThat(fileStats.value.getMin(), greaterThan(0L)); + assertThat(fileStats.value.getMax(), greaterThan(0L)); + } ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null); engine.index(indexForDoc(doc2)); engine.refresh("test"); - assertThat(engine.segmentsStats(true, false).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value)); + final SegmentsStats stats2 = engine.segmentsStats(true, false); + for (ObjectCursor cursor : stats1.getFiles().keys()) { + final String extension = cursor.value; + assertThat(stats2.getFiles().get(extension).getTotal(), greaterThan((stats1.getFiles().get(extension).getTotal()))); + assertThat(stats2.getFiles().get(extension).getCount(), greaterThan((stats1.getFiles().get(extension).getCount()))); + assertThat(stats2.getFiles().get(extension).getMin(), greaterThan((0L))); + assertThat(stats2.getFiles().get(extension).getMax(), greaterThan((0L))); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java index 17acc462b9faa..43bc1eaad9127 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -144,10 +144,10 @@ public void testNoOpEngineStats() throws Exception { assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes()); assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount()); // don't compare memory in bytes since we load the index with term-dict off-heap - assertEquals(expectedSegmentStats.getFileSizes().size(), - noOpEngine.segmentsStats(includeFileSize, true).getFileSizes().size()); + assertEquals(expectedSegmentStats.getFiles().size(), + noOpEngine.segmentsStats(includeFileSize, true).getFiles().size()); - assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getFileSizes().size()); + assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getFiles().size()); assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getMemoryInBytes()); } catch (AssertionError e) { logger.error(config.getMergePolicy()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java index 8f60c9168da1d..1418f13b55958 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java @@ -254,7 +254,7 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl final SegmentsStats stats = new SegmentsStats(); stats.add(this.segmentsStats); if (includeSegmentFileSizes == false) { - stats.clearFileSizes(); + stats.clearFiles(); } return stats; } else { diff --git a/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/indices_stats.yml b/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/indices_stats.yml new file mode 100644 index 0000000000000..6613fa664dfc2 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/indices_stats.yml @@ -0,0 +1,156 @@ +--- +setup: + - skip: + version: " - 7.99.99" + reason: segment files stats enhanced in 8.0.0 + + - do: + indices.create: + index: docs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + bulk: + body: + - index: + _index: docs + _id: 1 + - field: foo + - index: + _index: docs + _id: 2 + - field: bar + - index: + _index: docs + _id: 3 + - field: baz + + - do: + snapshot.create_repository: + repository: repository-fs + body: + type: fs + settings: + location: "repository-fs" + + # Remove the snapshot if a previous test failed to delete it. + # Useful for third party tests that runs the test against a real external service. + - do: + snapshot.delete: + repository: repository-fs + snapshot: snapshot + ignore: 404 + + - do: + snapshot.create: + repository: repository-fs + snapshot: snapshot + wait_for_completion: true + + - do: + indices.delete: + index: docs +--- +teardown: + + - do: + snapshot.delete: + repository: repository-fs + snapshot: snapshot + ignore: 404 + + - do: + snapshot.delete_repository: + repository: repository-fs + +--- +"Tests Indices Stats API for snapshot backed indices": + + - do: + searchable_snapshots.mount: + repository: repository-fs + snapshot: snapshot + wait_for_completion: true + body: + index: docs + renamed_index: cold-docs + + - match: { snapshot.snapshot: snapshot } + - match: { snapshot.shards.failed: 0 } + - match: { snapshot.shards.successful: 1 } + + - do: + searchable_snapshots.mount: + repository: repository-fs + snapshot: snapshot + wait_for_completion: true + storage: shared_cache + body: + index: docs + renamed_index: frozen-docs + + - match: { snapshot.snapshot: snapshot } + - match: { snapshot.shards.failed: 0 } + - match: { snapshot.shards.successful: 1 } + + - do: + indices.stats: + index: "*-docs" + metric: [ segments ] + include_segment_file_sizes: true + + - is_true: _all.total.segments.file_sizes + - is_true: _all.total.segments.file_sizes.si + - gt: { _all.total.segments.file_sizes.si.count: 0 } + - gt: { _all.total.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.average_size_in_bytes: 0 } + + - is_true: indices.cold-docs.primaries.segments.file_sizes + - is_true: indices.cold-docs.primaries.segments.file_sizes.si + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.count: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.average_size_in_bytes: 0 } + - match: { indices.cold-docs.primaries.segments.file_sizes.si.description: "Segment Info" } + + - is_true: indices.frozen-docs.primaries.segments.file_sizes + - is_false: indices.frozen-docs.primaries.segments.file_sizes.si + + - do: + indices.stats: + index: "*-docs" + metric: [ segments ] + include_segment_file_sizes: true + include_unloaded_segments: true + + - is_true: _all.total.segments.file_sizes + - is_true: _all.total.segments.file_sizes.si + - gt: { _all.total.segments.file_sizes.si.count: 0 } + - gt: { _all.total.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { _all.total.segments.file_sizes.si.average_size_in_bytes: 0 } + + - is_true: indices.cold-docs.primaries.segments.file_sizes + - is_true: indices.cold-docs.primaries.segments.file_sizes.si + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.count: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { indices.cold-docs.primaries.segments.file_sizes.si.average_size_in_bytes: 0 } + - match: { indices.cold-docs.primaries.segments.file_sizes.si.description: "Segment Info" } + + - is_true: indices.frozen-docs.primaries.segments.file_sizes + - is_true: indices.frozen-docs.primaries.segments.file_sizes.si + - gt: { indices.frozen-docs.primaries.segments.file_sizes.si.count: 0 } + - gt: { indices.frozen-docs.primaries.segments.file_sizes.si.size_in_bytes: 0 } + - gt: { indices.frozen-docs.primaries.segments.file_sizes.si.min_size_in_bytes: 0 } + - gt: { indices.frozen-docs.primaries.segments.file_sizes.si.max_size_in_bytes: 0 } + - gt: { indices.frozen-docs.primaries.segments.file_sizes.si.average_size_in_bytes: 0 } + - match: { indices.frozen-docs.primaries.segments.file_sizes.si.description: "Segment Info" }