From 7ea77924cc74694c6086c3a0b9b89bb61a39bf6f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:56:47 -0700 Subject: [PATCH] Fix index create block getting applied when there is no data nodes healthy (#6756) (#6771) (cherry picked from commit e6a370036928e147e07e8b8b119fe6086f3907d4) Signed-off-by: Rishav Sagar Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../allocation/DiskThresholdMonitor.java | 1 + .../allocation/DiskThresholdMonitorTests.java | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java index d93dc8b12d28a..6f63aff2f3a90 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java @@ -403,6 +403,7 @@ public void onNewInfo(ClusterInfo info) { // If all the nodes are breaching high disk watermark, we apply index create block to avoid red clusters. if ((state.getBlocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id()) == false) + && nodes.size() > 0 && nodesOverHighThreshold.size() == nodes.size()) { setIndexCreateBlock(listener, true); } else if (state.getBlocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id()) diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 4c4b362ef7c56..e4f3c4eeeb903 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -61,6 +61,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; @@ -681,6 +682,56 @@ protected void setIndexCreateBlock(ActionListener listener, boolean indexC ); } + public void testIndexCreateBlockWhenNoDataNodeHealthy() { + AllocationService allocation = createAllocationService( + Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build() + ); + Metadata metadata = Metadata.builder().build(); + RoutingTable routingTable = RoutingTable.builder().build(); + final ClusterState clusterState = applyStartedShardsUntilNoChange( + ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metadata(metadata) + .routingTable(routingTable) + .build(), + allocation + ); + AtomicInteger countBlocksCalled = new AtomicInteger(); + AtomicBoolean reroute = new AtomicBoolean(false); + AtomicReference> indices = new AtomicReference<>(); + AtomicLong currentTime = new AtomicLong(); + Settings settings = Settings.builder().build(); + DiskThresholdMonitor monitor = new DiskThresholdMonitor( + settings, + () -> clusterState, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + null, + currentTime::get, + (reason, priority, listener) -> { + assertTrue(reroute.compareAndSet(false, true)); + assertThat(priority, equalTo(Priority.HIGH)); + listener.onResponse(null); + } + ) { + + @Override + protected void updateIndicesReadOnly(Set indicesToMarkReadOnly, ActionListener listener, boolean readOnly) { + assertTrue(indices.compareAndSet(null, indicesToMarkReadOnly)); + assertFalse(readOnly); + listener.onResponse(null); + } + + @Override + protected void setIndexCreateBlock(ActionListener listener, boolean indexCreateBlock) { + countBlocksCalled.set(countBlocksCalled.get() + 1); + listener.onResponse(null); + } + }; + + ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + monitor.onNewInfo(clusterInfo(builder.build())); + assertTrue(countBlocksCalled.get() == 0); + } + private void assertNoLogging(DiskThresholdMonitor monitor, ImmutableOpenMap diskUsages) throws IllegalAccessException { try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(DiskThresholdMonitor.class))) {