From 8ba53b7001e8c3702099f0cc9d20b47c566545bf Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 29 Sep 2021 10:51:12 -0700 Subject: [PATCH] Revert "Removed multiple paths from MetadataStateFormat (#72821)" This reverts commit 76e515a279f3a332e276abe85030a318ac07c795. relates #71205 --- .../gateway/MetadataStateFormat.java | 157 +++++++++------ .../gateway/MetadataStateFormatTests.java | 188 +++++++++++------- .../index/shard/IndexShardTests.java | 8 +- 3 files changed, 219 insertions(+), 134 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index 34d48db543d46..cc37538eaf5d4 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -19,8 +19,8 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.NIOFSDirectory; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.core.Tuple; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.lucene.store.IndexOutputOutputStream; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -39,10 +39,12 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * MetadataStateFormat is a base class to write checksummed @@ -169,10 +171,10 @@ private static void performStateDirectoriesFsync(List> st /** * Writes the given state to the given directories and performs cleanup of old state files if the write succeeds or * newly created state file if write fails. - * See also {@link #write(Object, Path)} and {@link #cleanupOldFiles(long, Path)}. + * See also {@link #write(Object, Path...)} and {@link #cleanupOldFiles(long, Path[])}. */ - public final long writeAndCleanup(final T state, final Path location) throws WriteStateException { - return write(state, true, location); + public final long writeAndCleanup(final T state, final Path... locations) throws WriteStateException { + return write(state, true, locations); } /** @@ -187,26 +189,29 @@ public final long writeAndCleanup(final T state, final Path location) throws Wri * If this method fails with an exception, it performs cleanup of newly created state file. * But if this method succeeds, it does not perform cleanup of old state files. * If this write succeeds, but some further write fails, you may want to rollback the transaction and keep old file around. - * After transaction is finished use {@link #cleanupOldFiles(long, Path)} for the clean-up. - * If this write is not a part of bigger transaction, consider using {@link #writeAndCleanup(Object, Path)} method instead. + * After transaction is finished use {@link #cleanupOldFiles(long, Path[])} for the clean-up. + * If this write is not a part of bigger transaction, consider using {@link #writeAndCleanup(Object, Path...)} method instead. * * @param state the state object to write - * @param location the data dir the state should be written into + * @param locations the locations where the state should be written to. * @throws WriteStateException if some exception during writing state occurs. See also {@link WriteStateException#isDirty()}. * @return generation of newly written state. */ - public final long write(final T state, final Path location) throws WriteStateException { - return write(state, false, location); + public final long write(final T state, final Path... locations) throws WriteStateException { + return write(state, false, locations); } - private long write(final T state, boolean cleanup, final Path location) throws WriteStateException { - if (location == null) { + private long write(final T state, boolean cleanup, final Path... locations) throws WriteStateException { + if (locations == null) { throw new IllegalArgumentException("Locations must not be null"); } + if (locations.length <= 0) { + throw new IllegalArgumentException("One or more locations required"); + } final long oldGenerationId, newGenerationId; try { - oldGenerationId = findMaxGenerationId(prefix, location); + oldGenerationId = findMaxGenerationId(prefix, locations); newGenerationId = oldGenerationId + 1; } catch (Exception e) { throw new WriteStateException(false, "exception during looking up new generation id", e); @@ -218,11 +223,13 @@ private long write(final T state, boolean cleanup, final Path location) throws W List> directories = new ArrayList<>(); try { - Path stateLocation = location.resolve(STATE_DIR_NAME); - try { - directories.add(new Tuple<>(location, newDirectory(stateLocation))); - } catch (IOException e) { - throw new WriteStateException(false, "failed to open state directory " + stateLocation, e); + for (Path location : locations) { + Path stateLocation = location.resolve(STATE_DIR_NAME); + try { + directories.add(new Tuple<>(location, newDirectory(stateLocation))); + } catch (IOException e) { + throw new WriteStateException(false, "failed to open state directory " + stateLocation, e); + } } writeStateToFirstLocation(state, directories.get(0).v1(), directories.get(0).v2(), tmpFileName); @@ -231,7 +238,7 @@ private long write(final T state, boolean cleanup, final Path location) throws W performStateDirectoriesFsync(directories); } catch (WriteStateException e) { if (cleanup) { - cleanupOldFiles(oldGenerationId, location); + cleanupOldFiles(oldGenerationId, locations); } throw e; } finally { @@ -242,7 +249,7 @@ private long write(final T state, boolean cleanup, final Path location) throws W } if (cleanup) { - cleanupOldFiles(newGenerationId, location); + cleanupOldFiles(newGenerationId, locations); } return newGenerationId; @@ -309,20 +316,22 @@ protected Directory newDirectory(Path dir) throws IOException { * Clean ups all state files not matching passed generation. * * @param currentGeneration state generation to keep. - * @param location data dir. + * @param locations state paths. */ - public void cleanupOldFiles(final long currentGeneration, Path location) { + public void cleanupOldFiles(final long currentGeneration, Path... locations) { final String fileNameToKeep = getStateFileName(currentGeneration); - logger.trace("cleanupOldFiles: cleaning up {}", location); - Path stateLocation = location.resolve(STATE_DIR_NAME); - try (Directory stateDir = newDirectory(stateLocation)) { - for (String file : stateDir.listAll()) { - if (file.startsWith(prefix) && file.equals(fileNameToKeep) == false) { - deleteFileIgnoreExceptions(stateLocation, stateDir, file); + for (Path location : locations) { + logger.trace("cleanupOldFiles: cleaning up {}", location); + Path stateLocation = location.resolve(STATE_DIR_NAME); + try (Directory stateDir = newDirectory(stateLocation)) { + for (String file : stateDir.listAll()) { + if (file.startsWith(prefix) && file.equals(fileNameToKeep) == false) { + deleteFileIgnoreExceptions(stateLocation, stateDir, file); + } } + } catch (Exception e) { + logger.trace("clean up failed for state location {}", stateLocation); } - } catch (Exception e) { - logger.trace("clean up failed for state location {}", stateLocation); } } @@ -330,20 +339,22 @@ public void cleanupOldFiles(final long currentGeneration, Path location) { * Finds state file with maximum id. * * @param prefix - filename prefix - * @param dataLocation - path to directory with state folder + * @param locations - paths to directories with state folder * @return maximum id of state file or -1 if no such files are found * @throws IOException if IOException occurs */ - private long findMaxGenerationId(final String prefix, Path dataLocation) throws IOException { + private long findMaxGenerationId(final String prefix, Path... locations) throws IOException { long maxId = -1; - final Path resolve = dataLocation.resolve(STATE_DIR_NAME); - if (Files.exists(resolve)) { - try (DirectoryStream stream = Files.newDirectoryStream(resolve, prefix + "*")) { - for (Path stateFile : stream) { - final Matcher matcher = stateFilePattern.matcher(stateFile.getFileName().toString()); - if (matcher.matches()) { - final long id = Long.parseLong(matcher.group(1)); - maxId = Math.max(maxId, id); + for (Path dataLocation : locations) { + final Path resolve = dataLocation.resolve(STATE_DIR_NAME); + if (Files.exists(resolve)) { + try (DirectoryStream stream = Files.newDirectoryStream(resolve, prefix + "*")) { + for (Path stateFile : stream) { + final Matcher matcher = stateFilePattern.matcher(stateFile.getFileName().toString()); + if (matcher.matches()) { + final long id = Long.parseLong(matcher.group(1)); + maxId = Math.max(maxId, id); + } } } } @@ -351,19 +362,22 @@ private long findMaxGenerationId(final String prefix, Path dataLocation) throws return maxId; } - private Path findStateFilesByGeneration(final long generation, Path dataLocation) { + private List findStateFilesByGeneration(final long generation, Path... locations) { + List files = new ArrayList<>(); if (generation == -1) { - return null; + return files; } final String fileName = getStateFileName(generation); - final Path stateFilePath = dataLocation.resolve(STATE_DIR_NAME).resolve(fileName); - if (Files.exists(stateFilePath)) { - logger.trace("found state file: {}", stateFilePath); - return stateFilePath; + for (Path dataLocation : locations) { + final Path stateFilePath = dataLocation.resolve(STATE_DIR_NAME).resolve(fileName); + if (Files.exists(stateFilePath)) { + logger.trace("found state file: {}", stateFilePath); + files.add(stateFilePath); + } } - return null; + return files; } public String getStateFileName(long generation) { @@ -376,22 +390,31 @@ public String getStateFileName(long generation) { * * @param logger a logger instance. * @param generation the generation to be loaded. - * @param dataLocation the data dir to read from + * @param dataLocations the data-locations to try. * @return the state of asked generation or null if no state was found. */ - public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, long generation, Path dataLocation) { - Path stateFile = findStateFilesByGeneration(generation, dataLocation); + public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, long generation, Path... dataLocations) { + List stateFiles = findStateFilesByGeneration(generation, dataLocations); - if (stateFile != null) { + final List exceptions = new ArrayList<>(); + for (Path stateFile : stateFiles) { try { T state = read(namedXContentRegistry, stateFile); logger.trace("generation id [{}] read from [{}]", generation, stateFile.getFileName()); return state; } catch (Exception e) { - logger.debug(() -> new ParameterizedMessage("{}: failed to read [{}], ignoring...", stateFile, prefix), e); - throw new ElasticsearchException("failed to read " + stateFile, e); + exceptions.add(new IOException("failed to read " + stateFile, e)); + logger.debug(() -> new ParameterizedMessage( + "{}: failed to read [{}], ignoring...", stateFile, prefix), e); } } + // if we reach this something went wrong + ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptions); + if (stateFiles.size() > 0) { + // We have some state files but none of them gave us a usable state + throw new IllegalStateException("Could not find a state file to recover from among " + + stateFiles.stream().map(Object::toString).collect(Collectors.joining(", "))); + } return null; } @@ -399,17 +422,20 @@ public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegist * Tries to load the latest state from the given data-locations. * * @param logger a logger instance. - * @param dataLocation the data dir to read from + * @param dataLocations the data-locations to try. * @return tuple of the latest state and generation. (null, -1) if no state is found. */ - public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path dataLocation) + public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException { - long generation = findMaxGenerationId(prefix, dataLocation); - T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocation); + long generation = findMaxGenerationId(prefix, dataLocations); + T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocations); if (generation > -1 && state == null) { throw new IllegalStateException("unable to find state files with generation id " + generation + - " returned by findMaxGenerationId function, in data folder [" + dataLocation + "], concurrent writes?"); + " returned by findMaxGenerationId function, in data folders [" + + Arrays.stream(dataLocations). + map(Object::toString).collect(Collectors.joining(", ")) + + "], concurrent writes?"); } return Tuple.tuple(state, generation); } @@ -418,19 +444,24 @@ public Tuple loadLatestStateWithGeneration(Logger logger, NamedXContent * Tries to load the latest state from the given data-locations. * * @param logger a logger instance. - * @param dataLocation the data dir to read from + * @param dataLocations the data-locations to try. * @return the latest state or null if no state was found. */ - public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path dataLocation) throws IOException { - return loadLatestStateWithGeneration(logger, namedXContentRegistry, dataLocation).v1(); + public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws + IOException { + return loadLatestStateWithGeneration(logger, namedXContentRegistry, dataLocations).v1(); } /** * Deletes all meta state directories recursively for the given data locations - * @param dataLocation the data dir to delete state from + * @param dataLocations the data location to delete */ - public static void deleteMetaState(Path dataLocation) throws IOException { - IOUtils.rm(dataLocation.resolve(STATE_DIR_NAME)); + public static void deleteMetaState(Path... dataLocations) throws IOException { + Path[] stateDirectories = new Path[dataLocations.length]; + for (int i = 0; i < dataLocations.length; i++) { + stateDirectories[i] = dataLocations[i].resolve(STATE_DIR_NAME); + } + IOUtils.rm(stateDirectories); } public String getPrefix() { diff --git a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java index 791ffda50cb63..f28c53f1b9a6d 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetadataStateFormatTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Metadata; @@ -79,14 +80,17 @@ public Metadata fromXContent(XContentParser parser) throws IOException { } public void testReadWriteState() throws IOException { - final Path dir = createTempDir(); - final long id = addDummyFiles("foo-", dir); + Path[] dirs = new Path[randomIntBetween(1, 5)]; + for (int i = 0; i < dirs.length; i++) { + dirs[i] = createTempDir(); + } + final long id = addDummyFiles("foo-", dirs); Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.writeAndCleanup(state, dir); - { - Path[] list = content("*", dir); + format.writeAndCleanup(state, dirs); + for (Path file : dirs) { + Path[] list = content("*", file); assertEquals(list.length, 1); assertThat(list[0].getFileName().toString(), equalTo(MetadataStateFormat.STATE_DIR_NAME)); Path stateDir = list[0]; @@ -97,68 +101,79 @@ public void testReadWriteState() throws IOException { DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]); assertThat(read, equalTo(state)); } - DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), - randomDouble(), randomBoolean()); - format.writeAndCleanup(state2, dir); + DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), + randomDouble(), randomBoolean()); + format.writeAndCleanup(state2, dirs); - { - Path[] list = content("*", dir); + for (Path file : dirs) { + Path[] list = content("*", file); assertEquals(list.length, 1); assertThat(list[0].getFileName().toString(), equalTo(MetadataStateFormat.STATE_DIR_NAME)); Path stateDir = list[0]; assertThat(Files.isDirectory(stateDir), is(true)); list = content("foo-*", stateDir); - assertEquals(list.length, 1); - assertThat(list[0].getFileName().toString(), equalTo("foo-" + (id + 1) + ".st")); + assertEquals(list.length,1); + assertThat(list[0].getFileName().toString(), equalTo("foo-"+ (id+1) + ".st")); DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]); assertThat(read, equalTo(state2)); + } } public void testVersionMismatch() throws IOException { - final Path dir = createTempDir(); - final long id = addDummyFiles("foo-", dir); + Path[] dirs = new Path[randomIntBetween(1, 5)]; + for (int i = 0; i < dirs.length; i++) { + dirs[i] = createTempDir(); + } + final long id = addDummyFiles("foo-", dirs); Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.writeAndCleanup(state, dir); - Path[] list = content("*", dir); - assertEquals(list.length, 1); - assertThat(list[0].getFileName().toString(), equalTo(MetadataStateFormat.STATE_DIR_NAME)); - Path stateDir = list[0]; - assertThat(Files.isDirectory(stateDir), is(true)); - list = content("foo-*", stateDir); - assertEquals(list.length, 1); - assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st")); - DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]); - assertThat(read, equalTo(state)); + format.writeAndCleanup(state, dirs); + for (Path file : dirs) { + Path[] list = content("*", file); + assertEquals(list.length, 1); + assertThat(list[0].getFileName().toString(), equalTo(MetadataStateFormat.STATE_DIR_NAME)); + Path stateDir = list[0]; + assertThat(Files.isDirectory(stateDir), is(true)); + list = content("foo-*", stateDir); + assertEquals(list.length, 1); + assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st")); + DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]); + assertThat(read, equalTo(state)); + } } public void testCorruption() throws IOException { - final Path dir = createTempDir(); - final long id = addDummyFiles("foo-", dir); + Path[] dirs = new Path[randomIntBetween(1, 5)]; + for (int i = 0; i < dirs.length; i++) { + dirs[i] = createTempDir(); + } + final long id = addDummyFiles("foo-", dirs); Format format = new Format("foo-"); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.writeAndCleanup(state, dir); - Path[] list = content("*", dir); - assertEquals(list.length, 1); - assertThat(list[0].getFileName().toString(), equalTo(MetadataStateFormat.STATE_DIR_NAME)); - Path stateDir = list[0]; - assertThat(Files.isDirectory(stateDir), is(true)); - list = content("foo-*", stateDir); - assertEquals(list.length, 1); - assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st")); - DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]); - assertThat(read, equalTo(state)); - // now corrupt it - corruptFile(list[0], logger); - try { - format.read(NamedXContentRegistry.EMPTY, list[0]); - fail("corrupted file"); - } catch (CorruptStateException ex) { - // expected + format.writeAndCleanup(state, dirs); + for (Path file : dirs) { + Path[] list = content("*", file); + assertEquals(list.length, 1); + assertThat(list[0].getFileName().toString(), equalTo(MetadataStateFormat.STATE_DIR_NAME)); + Path stateDir = list[0]; + assertThat(Files.isDirectory(stateDir), is(true)); + list = content("foo-*", stateDir); + assertEquals(list.length, 1); + assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st")); + DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]); + assertThat(read, equalTo(state)); + // now corrupt it + corruptFile(list[0], logger); + try { + format.read(NamedXContentRegistry.EMPTY, list[0]); + fail("corrupted file"); + } catch (CorruptStateException ex) { + // expected + } } } @@ -203,15 +218,24 @@ public static void corruptFile(Path fileToCorrupt, Logger logger) throws IOExcep } } - private DummyState writeAndReadStateSuccessfully(Format format, Path path) throws IOException { + private DummyState writeAndReadStateSuccessfully(Format format, Path... paths) throws IOException { format.noFailures(); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); - format.writeAndCleanup(state, path); - assertEquals(state, format.loadLatestState(logger, NamedXContentRegistry.EMPTY, path)); + format.writeAndCleanup(state, paths); + assertEquals(state, format.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths)); + ensureOnlyOneStateFile(paths); return state; } + private static void ensureOnlyOneStateFile(Path[] paths) throws IOException { + for (Path path : paths) { + try (Directory dir = new NIOFSDirectory(path.resolve(MetadataStateFormat.STATE_DIR_NAME))) { + assertThat(dir.listAll().length, equalTo(1)); + } + } + } + public void testFailWriteAndReadPreviousState() throws IOException { Path path = createTempDir(); Format format = new Format("foo-"); @@ -256,12 +280,38 @@ public void testFailWriteAndReadAnyState() throws IOException { writeAndReadStateSuccessfully(format, path); } + public void testFailCopyTmpFileToExtraLocation() throws IOException { + Path paths[] = new Path[randomIntBetween(2, 5)]; + for (int i = 0; i < paths.length; i++) { + paths[i] = createTempDir(); + } + Format format = new Format("foo-"); + + DummyState initialState = writeAndReadStateSuccessfully(format, paths); + + for (int i = 0; i < randomIntBetween(1, 5); i++) { + format.failOnMethods(Format.FAIL_OPEN_STATE_FILE_WHEN_COPYING); + DummyState newState = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), + randomDouble(), randomBoolean()); + WriteStateException ex = expectThrows(WriteStateException.class, () -> format.writeAndCleanup(newState, paths)); + assertFalse(ex.isDirty()); + + format.noFailures(); + assertEquals(initialState, format.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths)); + } + + writeAndReadStateSuccessfully(format, paths); + } + public void testFailRandomlyAndReadAnyState() throws IOException { - Path path = createTempDir(); + Path paths[] = new Path[randomIntBetween(1, 5)]; + for (int i = 0; i < paths.length; i++) { + paths[i] = createTempDir(); + } Format format = new Format("foo-"); Set possibleStates = new HashSet<>(); - DummyState initialState = writeAndReadStateSuccessfully(format, path); + DummyState initialState = writeAndReadStateSuccessfully(format, paths); possibleStates.add(initialState); for (int i = 0; i < randomIntBetween(1, 5); i++) { @@ -269,7 +319,7 @@ public void testFailRandomlyAndReadAnyState() throws IOException { DummyState newState = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 100), randomInt(), randomLong(), randomDouble(), randomBoolean()); try { - format.writeAndCleanup(newState, path); + format.writeAndCleanup(newState, paths); possibleStates.clear(); possibleStates.add(newState); } catch (WriteStateException e) { @@ -279,17 +329,19 @@ public void testFailRandomlyAndReadAnyState() throws IOException { } format.noFailures(); - DummyState stateOnDisk = format.loadLatestState(logger, NamedXContentRegistry.EMPTY, path); + //we call loadLatestState not on full path set, but only on random paths from this set. This is to emulate disk failures. + Path[] randomPaths = randomSubsetOf(randomIntBetween(1, paths.length), paths).toArray(new Path[0]); + DummyState stateOnDisk = format.loadLatestState(logger, NamedXContentRegistry.EMPTY, randomPaths); assertTrue(possibleStates.contains(stateOnDisk)); if (possibleStates.size() > 1) { //if there was a WriteStateException we need to override current state before we continue - newState = writeAndReadStateSuccessfully(format, path); + newState = writeAndReadStateSuccessfully(format, paths); possibleStates.clear(); possibleStates.add(newState); } } - writeAndReadStateSuccessfully(format, path); + writeAndReadStateSuccessfully(format, paths); } private static class Format extends MetadataStateFormat { @@ -486,21 +538,23 @@ public Path[] content(String glob, Path dir) throws IOException { } } - public long addDummyFiles(String prefix, Path path) throws IOException { + public long addDummyFiles(String prefix, Path... paths) throws IOException { int realId = -1; - if (randomBoolean()) { - Path stateDir = path.resolve(MetadataStateFormat.STATE_DIR_NAME); - Files.createDirectories(stateDir); - String actualPrefix = prefix; - int id = randomIntBetween(0, 10); + for (Path path : paths) { if (randomBoolean()) { - actualPrefix = "dummy-"; - } else { - realId = Math.max(realId, id); - } - try (OutputStream stream = - Files.newOutputStream(stateDir.resolve(actualPrefix + id + MetadataStateFormat.STATE_FILE_EXTENSION))) { - stream.write(0); + Path stateDir = path.resolve(MetadataStateFormat.STATE_DIR_NAME); + Files.createDirectories(stateDir); + String actualPrefix = prefix; + int id = randomIntBetween(0, 10); + if (randomBoolean()) { + actualPrefix = "dummy-"; + } else { + realId = Math.max(realId, id); + } + try (OutputStream stream = + Files.newOutputStream(stateDir.resolve(actualPrefix + id + MetadataStateFormat.STATE_FILE_EXTENSION))) { + stream.write(0); + } } } return realId + 1; diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 0c0f7e43a9c87..123ef459fbdd0 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -189,13 +189,13 @@ */ public class IndexShardTests extends IndexShardTestCase { - public static ShardStateMetadata load(Logger logger, Path shardPath) throws IOException { - return ShardStateMetadata.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, shardPath); + public static ShardStateMetadata load(Logger logger, Path... shardPaths) throws IOException { + return ShardStateMetadata.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, shardPaths); } public static void write(ShardStateMetadata shardStateMetadata, - Path shardPath) throws IOException { - ShardStateMetadata.FORMAT.writeAndCleanup(shardStateMetadata, shardPath); + Path... shardPaths) throws IOException { + ShardStateMetadata.FORMAT.writeAndCleanup(shardStateMetadata, shardPaths); } public static Engine getEngineFromShard(IndexShard shard) {