Do refresh only for non system indices (#915) (#916)

Signed-off-by: Martin Gaievski <[email protected]> (cherry picked from commit a5214a3)
opensearch-project · May 25, 2023 · 591c3f9 · 591c3f9
1 parent 9a1606f
commit 591c3f9
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773))
 ### Bug Fixes
 ### Infrastructure
+* Disable index refresh for system indices ([#773](https://github.com/opensearch-project/k-NN/pull/915))
 ### Documentation
 ### Maintenance
 ### Refactoring
diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java
@@ -55,7 +55,7 @@ public static void setUpClass() throws IOException {
         testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath());
     }
 
-    public void testEndToEnd_fromMethod() throws IOException, InterruptedException {
+    public void testEndToEnd_fromMethod() throws Exception {
         String indexName = "test-index-1";
         String fieldName = "test-field-1";
 
@@ -106,7 +106,7 @@ public void testEndToEnd_fromMethod() throws IOException, InterruptedException {
         }
 
         // Assert we have the right number of documents in the index
-        refreshAllIndices();
+        refreshAllNonSystemIndices();
         assertEquals(testData.indexData.docs.length, getDocCount(indexName));
 
         int k = 10;

diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java
@@ -9,8 +9,12 @@
 import com.google.common.io.Resources;
 import com.google.common.primitives.Floats;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.util.EntityUtils;
 import org.opensearch.common.bytes.BytesReference;
 import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.knn.index.query.KNNQueryBuilder;
 import org.opensearch.knn.index.KNNSettings;
@@ -20,7 +24,6 @@
 import org.opensearch.knn.indices.ModelState;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.script.KNNScoringScriptEngine;
-import org.apache.http.util.EntityUtils;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.opensearch.client.Request;
@@ -59,6 +62,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.PriorityQueue;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -111,6 +115,7 @@ public class KNNRestTestCase extends ODFERestTestCase {
     private static final String DOCUMENT_FIELD_FOUND = "found";
     protected static final int DELAY_MILLI_SEC = 1000;
     protected static final int NUM_OF_ATTEMPTS = 30;
+    private static final String SYSTEM_INDEX_PREFIX = ".opendistro";
 
     @AfterClass
     public static void dumpCoverage() throws IOException, MalformedObjectNameException {
@@ -1266,4 +1271,39 @@ public interface IProxy {
 
         void reset();
     }
+
+    protected void refreshAllNonSystemIndices() throws Exception {
+        Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
+        XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue());
+        try (
+            XContentParser parser = xContentType.xContent()
+                .createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    response.getEntity().getContent()
+                )
+        ) {
+            XContentParser.Token token = parser.nextToken();
+            List<Map<String, Object>> parserList;
+            if (token == XContentParser.Token.START_ARRAY) {
+                parserList = parser.listOrderedMap().stream().map(obj -> (Map<String, Object>) obj).collect(Collectors.toList());
+            } else {
+                parserList = Collections.singletonList(parser.mapOrdered());
+            }
+            Set<String> indices = parserList.stream()
+                .map(index -> (String) index.get("index"))
+                .filter(index -> !index.startsWith(SYSTEM_INDEX_PREFIX))
+                .collect(Collectors.toSet());
+            for (String index : indices) {
+                refreshIndex(index);
+            }
+        }
+    }
+
+    protected void refreshIndex(final String index) throws IOException {
+        Request request = new Request("POST", "/" + index + "/_refresh");
+
+        Response response = client().performRequest(request);
+        assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode()));
+    }
 }