diff --git a/CHANGELOG.md b/CHANGELOG.md index c296f002a340b..aa68140715202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - The truncation limit of the OpenSearchJsonLayout logger is now configurable ([#6569](https://github.com/opensearch-project/OpenSearch/pull/6569)) - Add 'base_path' setting to File System Repository ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558)) - Return success on DeletePits when no PITs exist. ([#6544](https://github.com/opensearch-project/OpenSearch/pull/6544)) +- Add node repurpose command for search nodes ([#6517](https://github.com/opensearch-project/OpenSearch/pull/6517)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.18.0 to 2.20.0 ([#6490](https://github.com/opensearch-project/OpenSearch/pull/6490)) diff --git a/server/src/main/java/org/opensearch/env/NodeEnvironment.java b/server/src/main/java/org/opensearch/env/NodeEnvironment.java index 9a8605c252740..ef3e8aa7e5583 100644 --- a/server/src/main/java/org/opensearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/opensearch/env/NodeEnvironment.java @@ -1307,7 +1307,9 @@ static List collectFileCacheDataPath(NodePath fileCacheNodePath) throws IO for (Path indexPath : indexStream) { if (Files.isDirectory(indexPath)) { try (Stream shardStream = Files.list(indexPath)) { - shardStream.map(Path::toAbsolutePath).forEach(indexSubPaths::add); + shardStream.filter(NodeEnvironment::isShardPath) + .map(Path::toAbsolutePath) + .forEach(indexSubPaths::add); } } } diff --git a/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java b/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java index 58d6f33292273..35f2eead82159 100644 --- a/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java +++ b/server/src/main/java/org/opensearch/env/NodeRepurposeCommand.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -57,6 +58,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static org.opensearch.env.NodeEnvironment.CACHE_FOLDER; import static org.opensearch.env.NodeEnvironment.INDICES_FOLDER; /** @@ -68,12 +70,14 @@ public class NodeRepurposeCommand extends OpenSearchNodeCommand { static final String ABORTED_BY_USER_MSG = OpenSearchNodeCommand.ABORTED_BY_USER_MSG; static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = OpenSearchNodeCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG; - static final String NO_CLEANUP = "Node has node.data=true -> no clean up necessary"; + static final String NO_CLEANUP = "Node has node.data=true and node.search=true -> no clean up necessary"; static final String NO_DATA_TO_CLEAN_UP_FOUND = "No data to clean-up found"; static final String NO_SHARD_DATA_TO_CLEAN_UP_FOUND = "No shard data to clean-up found"; + static final String NO_FILE_CACHE_DATA_TO_CLEAN_UP_FOUND = "No file cache to clean-up found"; + private static final int FILE_CACHE_NODE_PATH_LOCATION = 0; public NodeRepurposeCommand() { - super("Repurpose this node to another cluster-manager/data role, cleaning up any excess persisted data"); + super("Repurpose this node to another cluster-manager/data/search role, cleaning up any excess persisted data"); } void testExecute(Terminal terminal, OptionSet options, Environment env) throws Exception { @@ -83,7 +87,7 @@ void testExecute(Terminal terminal, OptionSet options, Environment env) throws E @Override protected boolean validateBeforeLock(Terminal terminal, Environment env) { Settings settings = env.settings(); - if (DiscoveryNode.isDataNode(settings)) { + if (DiscoveryNode.isDataNode(settings) && DiscoveryNode.isSearchNode(settings)) { terminal.println(Terminal.Verbosity.NORMAL, NO_CLEANUP); return false; } @@ -94,85 +98,186 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) { @Override protected void processNodePaths(Terminal terminal, Path[] dataPaths, int nodeLockId, OptionSet options, Environment env) throws IOException { - assert DiscoveryNode.isDataNode(env.settings()) == false; + assert DiscoveryNode.isDataNode(env.settings()) == false || DiscoveryNode.isSearchNode(env.settings()) == false; + + boolean repurposeData = DiscoveryNode.isDataNode(env.settings()) == false; + boolean repurposeSearch = DiscoveryNode.isSearchNode(env.settings()) == false; if (DiscoveryNode.isClusterManagerNode(env.settings()) == false) { - processNoClusterManagerNoDataNode(terminal, dataPaths, env); + processNoClusterManagerRepurposeNode(terminal, dataPaths, env, repurposeData, repurposeSearch); } else { - processClusterManagerNoDataNode(terminal, dataPaths, env); + processClusterManagerRepurposeNode(terminal, dataPaths, env, repurposeData, repurposeSearch); } } - private void processNoClusterManagerNoDataNode(Terminal terminal, Path[] dataPaths, Environment env) throws IOException { + private void processNoClusterManagerRepurposeNode( + Terminal terminal, + Path[] dataPaths, + Environment env, + boolean repurposeData, + boolean repurposeSearch + ) throws IOException { NodeEnvironment.NodePath[] nodePaths = toNodePaths(dataPaths); + NodeEnvironment.NodePath fileCacheNodePath = toNodePaths(dataPaths)[FILE_CACHE_NODE_PATH_LOCATION]; + final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths); + final Metadata metadata = loadClusterState(terminal, env, persistedClusterStateService).metadata(); - terminal.println(Terminal.Verbosity.VERBOSE, "Collecting shard data paths"); - List shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePaths); + Set indexPaths = Set.of(); + List shardDataPaths = List.of(); + Set fileCachePaths = Set.of(); + List fileCacheDataPaths = List.of(); terminal.println(Terminal.Verbosity.VERBOSE, "Collecting index metadata paths"); List indexMetadataPaths = NodeEnvironment.collectIndexMetadataPaths(nodePaths); - Set indexPaths = uniqueParentPaths(shardDataPaths, indexMetadataPaths); + if (repurposeData) { + terminal.println(Terminal.Verbosity.VERBOSE, "Collecting shard data paths"); + shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePaths); + indexPaths = uniqueParentPaths(shardDataPaths, indexMetadataPaths); + } - final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths); + if (repurposeSearch) { + terminal.println(Terminal.Verbosity.VERBOSE, "Collecting file cache data paths"); + fileCacheDataPaths = NodeEnvironment.collectFileCacheDataPath(fileCacheNodePath); + fileCachePaths = uniqueParentPaths(fileCacheDataPaths, indexMetadataPaths); + } - final Metadata metadata = loadClusterState(terminal, env, persistedClusterStateService).metadata(); - if (indexPaths.isEmpty() && metadata.indices().isEmpty()) { + if (repurposeData && repurposeSearch && fileCacheDataPaths.isEmpty() && indexPaths.isEmpty() && metadata.indices().isEmpty()) { terminal.println(Terminal.Verbosity.NORMAL, NO_DATA_TO_CLEAN_UP_FOUND); return; + } else if (repurposeData && !repurposeSearch && indexPaths.isEmpty() && metadata.indices().isEmpty()) { + terminal.println(Terminal.Verbosity.NORMAL, NO_DATA_TO_CLEAN_UP_FOUND); + return; + } else if (!repurposeData && repurposeSearch && fileCacheDataPaths.isEmpty() && metadata.indices().isEmpty()) { + terminal.println(NO_FILE_CACHE_DATA_TO_CLEAN_UP_FOUND); + return; } final Set indexUUIDs = Sets.union( - indexUUIDsFor(indexPaths), - StreamSupport.stream(metadata.indices().values().spliterator(), false) - .map(imd -> imd.value.getIndexUUID()) - .collect(Collectors.toSet()) + indexUUIDsFor(fileCachePaths), + Sets.union( + indexUUIDsFor(indexPaths), + StreamSupport.stream(metadata.indices().values().spliterator(), false) + .map(imd -> imd.value.getIndexUUID()) + .collect(Collectors.toSet()) + ) ); - outputVerboseInformation(terminal, indexPaths, indexUUIDs, metadata); - - terminal.println(noClusterManagerMessage(indexUUIDs.size(), shardDataPaths.size(), indexMetadataPaths.size())); + List cleanUpPaths = new ArrayList<>(shardDataPaths); + cleanUpPaths.addAll(fileCacheDataPaths); + outputVerboseInformation(terminal, cleanUpPaths, indexUUIDs, metadata); + terminal.println(noClusterManagerMessage(indexUUIDs.size(), cleanUpPaths.size(), indexMetadataPaths.size())); outputHowToSeeVerboseInformation(terminal); - terminal.println("Node is being re-purposed as no-cluster-manager and no-data. Clean-up of index data will be performed."); + if (repurposeData && repurposeSearch) { + terminal.println( + "Node is being re-purposed as no-cluster-manager, no-data and no-search. Clean-up of index data and file cache will be performed." + ); + } else if (repurposeData) { + terminal.println("Node is being re-purposed as no-cluster-manager and no-data. Clean-up of index data will be performed."); + } else if (repurposeSearch) { + terminal.println( + "Node is being re-purposed as no-cluster-manager and no-search. Clean-up of file cache and corresponding index metadata will be performed." + ); + } confirm(terminal, "Do you want to proceed?"); - removePaths(terminal, indexPaths); // clean-up shard dirs // clean-up all metadata dirs MetadataStateFormat.deleteMetaState(dataPaths); - IOUtils.rm(Stream.of(dataPaths).map(path -> path.resolve(INDICES_FOLDER)).toArray(Path[]::new)); + if (repurposeData) { + removePaths(terminal, indexPaths); // clean-up shard dirs + IOUtils.rm(Stream.of(dataPaths).map(path -> path.resolve(INDICES_FOLDER)).toArray(Path[]::new)); + } + + if (repurposeSearch) { + removePaths(terminal, fileCachePaths); // clean-up file cache dirs + IOUtils.rm(dataPaths[FILE_CACHE_NODE_PATH_LOCATION].resolve(CACHE_FOLDER)); + } - terminal.println("Node successfully repurposed to no-cluster-manager and no-data."); + if (repurposeData && repurposeSearch) { + terminal.println("Node successfully repurposed to no-cluster-manager, no-data and no-search."); + } else if (repurposeData) { + terminal.println("Node successfully repurposed to no-cluster-manager and no-data."); + } else if (repurposeSearch) { + terminal.println("Node successfully repurposed to no-cluster-manager and no-search."); + } } - private void processClusterManagerNoDataNode(Terminal terminal, Path[] dataPaths, Environment env) throws IOException { + private void processClusterManagerRepurposeNode( + Terminal terminal, + Path[] dataPaths, + Environment env, + boolean repurposeData, + boolean repurposeSearch + ) throws IOException { NodeEnvironment.NodePath[] nodePaths = toNodePaths(dataPaths); + NodeEnvironment.NodePath fileCacheNodePath = toNodePaths(dataPaths)[FILE_CACHE_NODE_PATH_LOCATION]; + final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths); + final Metadata metadata = loadClusterState(terminal, env, persistedClusterStateService).metadata(); - terminal.println(Terminal.Verbosity.VERBOSE, "Collecting shard data paths"); - List shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePaths); - if (shardDataPaths.isEmpty()) { - terminal.println(NO_SHARD_DATA_TO_CLEAN_UP_FOUND); - return; - } + Set indexPaths = Set.of(); + List shardDataPaths = List.of(); + Set fileCachePaths = Set.of(); + List fileCacheDataPaths = List.of(); - final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths); + if (repurposeData) { + terminal.println(Terminal.Verbosity.VERBOSE, "Collecting shard data paths"); + shardDataPaths = NodeEnvironment.collectShardDataPaths(nodePaths); + indexPaths = uniqueParentPaths(shardDataPaths); + } - final Metadata metadata = loadClusterState(terminal, env, persistedClusterStateService).metadata(); + if (repurposeSearch) { + terminal.println(Terminal.Verbosity.VERBOSE, "Collecting file cache data paths"); + fileCacheDataPaths = NodeEnvironment.collectFileCacheDataPath(fileCacheNodePath); + fileCachePaths = uniqueParentPaths(fileCacheDataPaths); + } - final Set indexPaths = uniqueParentPaths(shardDataPaths); - final Set indexUUIDs = indexUUIDsFor(indexPaths); + if (repurposeData && repurposeSearch && shardDataPaths.isEmpty() && fileCacheDataPaths.isEmpty()) { + terminal.println(NO_SHARD_DATA_TO_CLEAN_UP_FOUND); + return; + } else if (repurposeData && !repurposeSearch && shardDataPaths.isEmpty()) { + terminal.println(NO_SHARD_DATA_TO_CLEAN_UP_FOUND); + return; + } else if (!repurposeData && repurposeSearch && fileCacheDataPaths.isEmpty()) { + terminal.println(NO_FILE_CACHE_DATA_TO_CLEAN_UP_FOUND); + return; + } - outputVerboseInformation(terminal, shardDataPaths, indexUUIDs, metadata); + final Set indexUUIDs = Sets.union(indexUUIDsFor(indexPaths), indexUUIDsFor(fileCachePaths)); - terminal.println(shardMessage(shardDataPaths.size(), indexUUIDs.size())); + List cleanUpPaths = new ArrayList<>(shardDataPaths); + cleanUpPaths.addAll(fileCacheDataPaths); + outputVerboseInformation(terminal, cleanUpPaths, indexUUIDs, metadata); + terminal.println(shardMessage(cleanUpPaths.size(), indexUUIDs.size())); outputHowToSeeVerboseInformation(terminal); - terminal.println("Node is being re-purposed as cluster-manager and no-data. Clean-up of shard data will be performed."); + if (repurposeData && repurposeSearch) { + terminal.println( + "Node is being re-purposed as cluster-manager, no-data and no-search. Clean-up of shard data and file cache data will be performed." + ); + } else if (repurposeData) { + terminal.println("Node is being re-purposed as cluster-manager and no-data. Clean-up of shard data will be performed."); + } else if (repurposeSearch) { + terminal.println("Node is being re-purposed as cluster-manager and no-search. Clean-up of file cache data will be performed."); + } + confirm(terminal, "Do you want to proceed?"); - removePaths(terminal, shardDataPaths); // clean-up shard dirs + if (repurposeData) { + removePaths(terminal, shardDataPaths); // clean-up shard dirs + } + + if (repurposeSearch) { + removePaths(terminal, fileCacheDataPaths); // clean-up file cache dirs + } - terminal.println("Node successfully repurposed to cluster-manager and no-data."); + if (repurposeData && repurposeSearch) { + terminal.println("Node successfully repurposed to cluster-manager, no-data and no-search."); + } else if (repurposeData) { + terminal.println("Node successfully repurposed to cluster-manager and no-data."); + } else if (repurposeSearch) { + terminal.println("Node successfully repurposed to cluster-manager and no-search."); + } } private ClusterState loadClusterState(Terminal terminal, Environment env, PersistedClusterStateService psf) throws IOException { @@ -211,11 +316,17 @@ private Set indexUUIDsFor(Set indexPaths) { } static String noClusterManagerMessage(int indexes, int shards, int indexMetadata) { - return "Found " + indexes + " indices (" + shards + " shards and " + indexMetadata + " index meta data) to clean up"; + return "Found " + + indexes + + " indices (" + + shards + + " shards/file cache folders and " + + indexMetadata + + " index meta data) to clean up"; } static String shardMessage(int shards, int indices) { - return "Found " + shards + " shards in " + indices + " indices to clean up"; + return "Found " + shards + " shards/file cache folders in " + indices + " indices to clean up"; } private void removePaths(Terminal terminal, Collection paths) { diff --git a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java index 009ff324809b4..f172ebe7d37d9 100644 --- a/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java +++ b/server/src/test/java/org/opensearch/env/NodeRepurposeCommandTests.java @@ -46,6 +46,8 @@ import org.opensearch.common.CheckedRunnable; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.ByteSizeUnit; +import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.util.BigArrays; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.index.Index; @@ -57,16 +59,19 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; +import java.util.Set; import java.util.stream.Stream; import static org.opensearch.env.NodeRepurposeCommand.NO_CLEANUP; import static org.opensearch.env.NodeRepurposeCommand.NO_DATA_TO_CLEAN_UP_FOUND; +import static org.opensearch.env.NodeRepurposeCommand.NO_FILE_CACHE_DATA_TO_CLEAN_UP_FOUND; import static org.opensearch.env.NodeRepurposeCommand.NO_SHARD_DATA_TO_CLEAN_UP_FOUND; +import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING; +import static org.opensearch.test.NodeRoles.addRoles; import static org.opensearch.test.NodeRoles.clusterManagerNode; import static org.opensearch.test.NodeRoles.nonDataNode; import static org.opensearch.test.NodeRoles.nonClusterManagerNode; +import static org.opensearch.test.NodeRoles.onlyRole; import static org.opensearch.test.NodeRoles.removeRoles; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -76,15 +81,35 @@ public class NodeRepurposeCommandTests extends OpenSearchTestCase { private static final Index INDEX = new Index("testIndex", "testUUID"); private Settings dataClusterManagerSettings; + private Settings dataSearchClusterManagerSettings; private Environment environment; private Path[] nodePaths; - private Settings dataNoClusterManagerSettings; + private Settings dataSearchNoClusterManagerSettings; private Settings noDataNoClusterManagerSettings; private Settings noDataClusterManagerSettings; + private Settings searchNoDataNoClusterManagerSettings; + private Settings noSearchNoClusterManagerSettings; @Before public void createNodePaths() throws IOException { dataClusterManagerSettings = buildEnvSettings(Settings.EMPTY); + Settings defaultSearchSettings = Settings.builder() + .put(dataClusterManagerSettings) + .put(NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(16, ByteSizeUnit.GB)) + .build(); + + searchNoDataNoClusterManagerSettings = onlyRole(dataClusterManagerSettings, DiscoveryNodeRole.SEARCH_ROLE); + dataSearchClusterManagerSettings = addRoles(defaultSearchSettings, Set.of(DiscoveryNodeRole.SEARCH_ROLE)); + noDataClusterManagerSettings = clusterManagerNode(nonDataNode(dataClusterManagerSettings)); + + dataSearchNoClusterManagerSettings = nonClusterManagerNode(dataSearchClusterManagerSettings); + noSearchNoClusterManagerSettings = nonClusterManagerNode(defaultSearchSettings); + + noDataNoClusterManagerSettings = removeRoles( + dataClusterManagerSettings, + Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE) + ); + environment = TestEnvironment.newEnvironment(dataClusterManagerSettings); try (NodeEnvironment nodeEnvironment = new NodeEnvironment(dataClusterManagerSettings, environment)) { nodePaths = nodeEnvironment.nodeDataPaths(); @@ -102,20 +127,13 @@ public void createNodePaths() throws IOException { writer.writeFullStateAndCommit(1L, ClusterState.EMPTY_STATE); } } - dataNoClusterManagerSettings = nonClusterManagerNode(dataClusterManagerSettings); - noDataNoClusterManagerSettings = removeRoles( - dataClusterManagerSettings, - Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.CLUSTER_MANAGER_ROLE))) - ); - - noDataClusterManagerSettings = clusterManagerNode(nonDataNode(dataClusterManagerSettings)); } public void testEarlyExitNoCleanup() throws Exception { createIndexDataFiles(dataClusterManagerSettings, randomInt(10), randomBoolean()); - verifyNoQuestions(dataClusterManagerSettings, containsString(NO_CLEANUP)); - verifyNoQuestions(dataNoClusterManagerSettings, containsString(NO_CLEANUP)); + verifyNoQuestions(dataSearchClusterManagerSettings, containsString(NO_CLEANUP)); + verifyNoQuestions(dataSearchNoClusterManagerSettings, containsString(NO_CLEANUP)); } public void testNothingToCleanup() throws Exception { @@ -138,6 +156,7 @@ public void testNothingToCleanup() throws Exception { verifyNoQuestions(noDataNoClusterManagerSettings, containsString(NO_DATA_TO_CLEAN_UP_FOUND)); verifyNoQuestions(noDataClusterManagerSettings, containsString(NO_SHARD_DATA_TO_CLEAN_UP_FOUND)); + verifyNoQuestions(noSearchNoClusterManagerSettings, containsString(NO_FILE_CACHE_DATA_TO_CLEAN_UP_FOUND)); createIndexDataFiles(dataClusterManagerSettings, 0, randomBoolean()); @@ -159,7 +178,7 @@ public void testLocked() throws IOException { } } - public void testCleanupAll() throws Exception { + public void testCleanupDataClusterManager() throws Exception { int shardCount = randomIntBetween(1, 10); boolean verbose = randomBoolean(); boolean hasClusterState = randomBoolean(); @@ -208,6 +227,84 @@ public void testCleanupShardData() throws Exception { new NodeEnvironment(noDataClusterManagerSettings, environment).close(); } + public void testCleanupSearchNode() throws Exception { + int shardCount = randomIntBetween(1, 10); + boolean verbose = randomBoolean(); + boolean hasClusterState = randomBoolean(); + createIndexDataFiles(searchNoDataNoClusterManagerSettings, shardCount, hasClusterState, true); + + Matcher matcher = allOf( + containsString(NodeRepurposeCommand.shardMessage(shardCount, 1)), + conditionalNot(containsString("testUUID"), verbose == false), + conditionalNot(containsString("testIndex"), verbose == false || hasClusterState == false), + conditionalNot(containsString("no name for uuid: testUUID"), verbose == false || hasClusterState) + ); + + verifyUnchangedOnAbort(dataClusterManagerSettings, matcher, verbose); + + // verify test setup + expectThrows(IllegalStateException.class, () -> new NodeEnvironment(dataClusterManagerSettings, environment).close()); + + verifySuccess(dataClusterManagerSettings, matcher, verbose); + + // verify clean. + new NodeEnvironment(dataClusterManagerSettings, environment).close(); + } + + public void testCleanupSearchClusterManager() throws Exception { + int shardCount = randomIntBetween(1, 10); + boolean verbose = randomBoolean(); + boolean hasClusterState = randomBoolean(); + createIndexDataFiles(dataSearchClusterManagerSettings, shardCount, hasClusterState, true); + + String messageText = NodeRepurposeCommand.noClusterManagerMessage(1, shardCount, 0); + + Matcher matcher = allOf( + containsString(messageText), + conditionalNot(containsString("testUUID"), verbose == false), + conditionalNot(containsString("testIndex"), verbose == false || hasClusterState == false), + conditionalNot(containsString("no name for uuid: testUUID"), verbose == false || hasClusterState) + ); + + verifyUnchangedOnAbort(noSearchNoClusterManagerSettings, matcher, verbose); + + // verify test setup + expectThrows(IllegalStateException.class, () -> new NodeEnvironment(noSearchNoClusterManagerSettings, environment).close()); + + verifySuccess(noSearchNoClusterManagerSettings, matcher, verbose); + + // verify clean. + new NodeEnvironment(noSearchNoClusterManagerSettings, environment).close(); + } + + public void testCleanupAll() throws Exception { + int shardCount = randomIntBetween(1, 10); + boolean verbose = randomBoolean(); + boolean hasClusterState = randomBoolean(); + createIndexDataFiles(dataSearchClusterManagerSettings, shardCount, hasClusterState, false); + createIndexDataFiles(dataSearchClusterManagerSettings, shardCount, hasClusterState, true); + + // environment.dataFiles().length * shardCount will account for the local shard files + // + shardCount will account for the additional file cache shard files. + String messageText = NodeRepurposeCommand.noClusterManagerMessage(1, (environment.dataFiles().length * shardCount) + shardCount, 0); + + Matcher outputMatcher = allOf( + containsString(messageText), + conditionalNot(containsString("testIndex"), verbose == false || hasClusterState == false), + conditionalNot(containsString("no name for uuid: testUUID"), verbose == false || hasClusterState) + ); + + verifyUnchangedOnAbort(noDataNoClusterManagerSettings, outputMatcher, verbose); + + // verify test setup + expectThrows(IllegalStateException.class, () -> new NodeEnvironment(noDataNoClusterManagerSettings, environment).close()); + + verifySuccess(noDataNoClusterManagerSettings, outputMatcher, verbose); + + // verify cleaned. + new NodeEnvironment(noDataNoClusterManagerSettings, environment).close(); + } + static void verifySuccess(Settings settings, Matcher outputMatcher, boolean verbose) throws Exception { withTerminal(verbose, outputMatcher, terminal -> { terminal.addTextInput(randomFrom("y", "Y")); @@ -256,6 +353,10 @@ private static void executeRepurposeCommand(MockTerminal terminal, Settings sett } private void createIndexDataFiles(Settings settings, int shardCount, boolean writeClusterState) throws IOException { + createIndexDataFiles(settings, shardCount, writeClusterState, false); + } + + private void createIndexDataFiles(Settings settings, int shardCount, boolean writeClusterState, boolean cacheMode) throws IOException { int shardDataDirNumber = randomInt(10); Environment environment = TestEnvironment.newEnvironment(settings); try (NodeEnvironment env = new NodeEnvironment(settings, environment)) { @@ -287,12 +388,23 @@ private void createIndexDataFiles(Settings settings, int shardCount, boolean wri ); } } - for (Path path : env.indexPaths(INDEX)) { + + if (cacheMode) { + Path cachePath = env.fileCacheNodePath().fileCachePath; + cachePath = cachePath.resolve(String.valueOf(env.getNodeLockId())).resolve(INDEX.getUUID()); for (int i = 0; i < shardCount; ++i) { - Files.createDirectories(path.resolve(Integer.toString(shardDataDirNumber))); + Files.createDirectories(cachePath.resolve(Integer.toString(shardDataDirNumber))); shardDataDirNumber += randomIntBetween(1, 10); } + } else { + for (Path path : env.indexPaths(INDEX)) { + for (int i = 0; i < shardCount; ++i) { + Files.createDirectories(path.resolve(Integer.toString(shardDataDirNumber))); + shardDataDirNumber += randomIntBetween(1, 10); + } + } } + } }