From 65385426033fe105df8aee61d97d7d92b4ab0ecf Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 12 Feb 2018 08:07:01 -0700 Subject: [PATCH] Switch to hardcoding Smile as the state format (#28610) This commit changes the state format that was previously passed in to `MetaDataStateFormat` to always use Smile. This doesn't actually change the format, since we have used Smile for writing the format since at least 5.0. This removes the automatic detection of the state format when reading state, since any state that could be processed in 6.x and 7.x would already have been written in Smile format. This is work towards removing the deprecated methods in the XContent code where we do automatic content-type detection. Relates to #28504 --- .../cluster/metadata/IndexMetaData.java | 2 +- .../cluster/metadata/MetaData.java | 2 +- .../org/elasticsearch/env/NodeMetaData.java | 2 +- .../gateway/MetaDataStateFormat.java | 23 ++++++++----------- .../index/shard/ShardStateMetaData.java | 2 +- .../gateway/MetaDataStateFormatTests.java | 21 ++++++++--------- 6 files changed, 23 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index b532a1ec841bb..414e06a236511 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1293,7 +1293,7 @@ public static Settings addHumanReadableSettings(Settings settings) { /** * State format for {@link IndexMetaData} to write to and load from disk */ - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.SMILE, INDEX_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(INDEX_STATE_FILE_PREFIX) { @Override public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 23ed28569d28d..8c6829ca78734 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -1205,7 +1205,7 @@ public static MetaData fromXContent(XContentParser parser) throws IOException { /** * State format for {@link MetaData} to write to and load from disk */ - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(GLOBAL_STATE_FILE_PREFIX) { @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/env/NodeMetaData.java b/server/src/main/java/org/elasticsearch/env/NodeMetaData.java index 38a4fce9cdc3d..dbea3164c8a44 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeMetaData.java +++ b/server/src/main/java/org/elasticsearch/env/NodeMetaData.java @@ -91,7 +91,7 @@ public NodeMetaData build() { } - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.SMILE, "node-") { + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat("node-") { @Override protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java index fb48405b72538..372e13f98acc0 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java @@ -64,36 +64,28 @@ * @param the type of the XContent base data-structure */ public abstract class MetaDataStateFormat { + public static final XContentType FORMAT = XContentType.SMILE; public static final String STATE_DIR_NAME = "_state"; public static final String STATE_FILE_EXTENSION = ".st"; + private static final String STATE_FILE_CODEC = "state"; private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 0; private static final int STATE_FILE_VERSION = 1; private static final int STATE_FILE_VERSION_ES_2X_AND_BELOW = 0; private static final int BUFFER_SIZE = 4096; - private final XContentType format; private final String prefix; private final Pattern stateFilePattern; /** * Creates a new {@link MetaDataStateFormat} instance - * @param format the format of the x-content */ - protected MetaDataStateFormat(XContentType format, String prefix) { - this.format = format; + protected MetaDataStateFormat(String prefix) { this.prefix = prefix; this.stateFilePattern = Pattern.compile(Pattern.quote(prefix) + "(\\d+)(" + MetaDataStateFormat.STATE_FILE_EXTENSION + ")?"); } - /** - * Returns the {@link XContentType} used to serialize xcontent on write. - */ - public XContentType format() { - return format; - } - /** * Writes the given state to the given directories. The state is written to a * state directory ({@value #STATE_DIR_NAME}) underneath each of the given file locations and is created if it @@ -123,8 +115,8 @@ public final void write(final T state, final Path... locations) throws IOExcepti try (OutputStreamIndexOutput out = new OutputStreamIndexOutput(resourceDesc, fileName, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) { CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION); - out.writeInt(format.index()); - try (XContentBuilder builder = newXContentBuilder(format, new IndexOutputOutputStream(out) { + out.writeInt(FORMAT.index()); + try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) { @Override public void close() throws IOException { // this is important since some of the XContentBuilders write bytes on close. @@ -190,6 +182,9 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION); final XContentType xContentType = XContentType.values()[indexInput.readInt()]; + if (xContentType != FORMAT) { + throw new IllegalStateException("expected state in " + file + " to be " + FORMAT + " format but was " + xContentType); + } if (fileVersion == STATE_FILE_VERSION_ES_2X_AND_BELOW) { // format version 0, wrote a version that always came from the content state file and was never used indexInput.readLong(); // version currently unused @@ -197,7 +192,7 @@ public final T read(NamedXContentRegistry namedXContentRegistry, Path file) thro long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) { - try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(namedXContentRegistry, + try (XContentParser parser = XContentFactory.xContent(FORMAT).createParser(namedXContentRegistry, new InputStreamIndexInput(slice, contentSize))) { return fromXContent(parser); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java b/server/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java index a1133d565f754..3f3f2a78100af 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java @@ -88,7 +88,7 @@ public String toString() { return "primary [" + primary + "], allocation [" + allocationId + "]"; } - public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.SMILE, SHARD_STATE_FILE_PREFIX) { + public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(SHARD_STATE_FILE_PREFIX) { @Override protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java index e82c5fafbf31a..543b5e39026fb 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java @@ -74,7 +74,7 @@ public class MetaDataStateFormatTests extends ESTestCase { * Ensure we can read a pre-generated cluster state. */ public void testReadClusterState() throws URISyntaxException, IOException { - final MetaDataStateFormat format = new MetaDataStateFormat(randomFrom(XContentType.values()), "global-") { + final MetaDataStateFormat format = new MetaDataStateFormat("global-") { @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { @@ -103,7 +103,7 @@ public void testReadWriteState() throws IOException { dirs[i] = createTempDir(); } final long id = addDummyFiles("foo-", dirs); - Format format = new Format(randomFrom(XContentType.values()), "foo-"); + Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); format.write(state, dirs); @@ -145,7 +145,7 @@ public void testVersionMismatch() throws IOException { } final long id = addDummyFiles("foo-", dirs); - Format format = new Format(randomFrom(XContentType.values()), "foo-"); + Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); format.write(state, dirs); @@ -169,7 +169,7 @@ public void testCorruption() throws IOException { dirs[i] = createTempDir(); } final long id = addDummyFiles("foo-", dirs); - Format format = new Format(randomFrom(XContentType.values()), "foo-"); + Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); format.write(state, dirs); @@ -244,17 +244,16 @@ public void testLoadState() throws IOException { meta.add(randomMeta()); } Set corruptedFiles = new HashSet<>(); - MetaDataStateFormat format = metaDataFormat(randomFrom(XContentType.values())); + MetaDataStateFormat format = metaDataFormat(); for (int i = 0; i < dirs.length; i++) { dirs[i] = createTempDir(); Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME)); for (int j = 0; j < numLegacy; j++) { - XContentType type = format.format(); if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j); Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them } else { - try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, + try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(MetaDataStateFormat.FORMAT, Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) { xcontentBuilder.startObject(); MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, ToXContent.EMPTY_PARAMS); @@ -309,8 +308,8 @@ public void testLoadState() throws IOException { } } - private static MetaDataStateFormat metaDataFormat(XContentType format) { - return new MetaDataStateFormat(format, MetaData.GLOBAL_STATE_FILE_PREFIX) { + private static MetaDataStateFormat metaDataFormat() { + return new MetaDataStateFormat(MetaData.GLOBAL_STATE_FILE_PREFIX) { @Override public void toXContent(XContentBuilder builder, MetaData state) throws IOException { MetaData.Builder.toXContent(state, builder, ToXContent.EMPTY_PARAMS); @@ -347,8 +346,8 @@ private IndexMetaData.Builder indexBuilder(String index) throws IOException { private class Format extends MetaDataStateFormat { - Format(XContentType format, String prefix) { - super(format, prefix); + Format(String prefix) { + super(prefix); } @Override