-
Notifications
You must be signed in to change notification settings - Fork 127
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Backport 2.x] [k-NN] Add Clear Cache API (#1055)
* [k-NN] Add Clear Cache API (#740) * Add Clear Cache API Signed-off-by: Naveen Tatikonda <[email protected]> * Add Unit and Integration tests Signed-off-by: Naveen Tatikonda <[email protected]> * Add BWC Tests Signed-off-by: Naveen Tatikonda <[email protected]> * Add CHANGELOG Signed-off-by: Naveen Tatikonda <[email protected]> * Address Review Comments Signed-off-by: Naveen Tatikonda <[email protected]> --------- Signed-off-by: Naveen Tatikonda <[email protected]> (cherry picked from commit 12f4a51) Signed-off-by: Naveen Tatikonda <[email protected]> * Fix Failing Clear Cache API Test (#1336) Signed-off-by: Naveen Tatikonda <[email protected]> --------- Signed-off-by: Naveen Tatikonda <[email protected]> Co-authored-by: Naveen Tatikonda <[email protected]>
- Loading branch information
1 parent
cee100f
commit 78a489a
Showing
17 changed files
with
842 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
49 changes: 49 additions & 0 deletions
49
qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.bwc; | ||
|
|
||
| import java.util.Collections; | ||
| import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; | ||
|
|
||
| public class ClearCacheIT extends AbstractRestartUpgradeTestCase { | ||
| private static final String TEST_FIELD = "test-field"; | ||
| private static final int DIMENSIONS = 5; | ||
| private static int docId = 0; | ||
| private static final int NUM_DOCS = 10; | ||
| private static int queryCnt = 0; | ||
| private static final int K = 5; | ||
|
|
||
| // Restart Upgrade BWC Tests to validate Clear Cache API | ||
| public void testClearCache() throws Exception { | ||
| waitForClusterHealthGreen(NODES_BWC_CLUSTER); | ||
| if (isRunningAgainstOldCluster()) { | ||
| createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); | ||
| } else { | ||
| queryCnt = NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
|
|
||
| docId = NUM_DOCS; | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); | ||
|
|
||
| queryCnt = queryCnt + NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
| deleteKNNIndex(testIndex); | ||
| } | ||
| } | ||
|
|
||
| // validation steps for Clear Cache API after upgrading node to new version | ||
| private void validateClearCacheOnUpgrade(int queryCount) throws Exception { | ||
| int graphCount = getTotalGraphsInCache(); | ||
| knnWarmup(Collections.singletonList(testIndex)); | ||
| assertTrue(getTotalGraphsInCache() > graphCount); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
|
|
||
| clearCache(Collections.singletonList(testIndex)); | ||
| assertEquals(0, getTotalGraphsInCache()); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
| } | ||
| } |
55 changes: 55 additions & 0 deletions
55
qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.bwc; | ||
|
|
||
| import java.util.Collections; | ||
|
|
||
| import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; | ||
|
|
||
| public class ClearCacheIT extends AbstractRollingUpgradeTestCase { | ||
| private static final String TEST_FIELD = "test-field"; | ||
| private static final int DIMENSIONS = 5; | ||
| private static int docId = 0; | ||
| private static final int K = 5; | ||
| private static final int NUM_DOCS = 10; | ||
| private static int queryCnt = 0; | ||
|
|
||
| // Rolling Upgrade BWC Tests to validate Clear Cache API | ||
| public void testClearCache() throws Exception { | ||
| waitForClusterHealthGreen(NODES_BWC_CLUSTER); | ||
| switch (getClusterType()) { | ||
| case OLD: | ||
| createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); | ||
| int docIdOld = 0; | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); | ||
| break; | ||
| case UPGRADED: | ||
| queryCnt = NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
|
|
||
| docId = NUM_DOCS; | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); | ||
|
|
||
| queryCnt = queryCnt + NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
| deleteKNNIndex(testIndex); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| // validation steps for Clear Cache API after upgrading all nodes from old version to new version | ||
| public void validateClearCacheOnUpgrade(int queryCount) throws Exception { | ||
| int graphCount = getTotalGraphsInCache(); | ||
| knnWarmup(Collections.singletonList(testIndex)); | ||
| assertTrue(getTotalGraphsInCache() > graphCount); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
|
|
||
| clearCache(Collections.singletonList(testIndex)); | ||
| assertEquals(0, getTotalGraphsInCache()); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
98 changes: 98 additions & 0 deletions
98
src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.plugin.rest; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.extern.log4j.Log4j2; | ||
| import org.opensearch.client.node.NodeClient; | ||
| import org.opensearch.cluster.metadata.IndexNameExpressionResolver; | ||
| import org.opensearch.cluster.service.ClusterService; | ||
| import org.opensearch.core.common.Strings; | ||
| import org.opensearch.core.index.Index; | ||
| import org.opensearch.knn.common.exception.KNNInvalidIndicesException; | ||
| import org.opensearch.knn.plugin.KNNPlugin; | ||
| import org.opensearch.knn.plugin.transport.ClearCacheAction; | ||
| import org.opensearch.knn.plugin.transport.ClearCacheRequest; | ||
| import org.opensearch.rest.BaseRestHandler; | ||
| import org.opensearch.rest.RestRequest; | ||
| import org.opensearch.rest.action.RestToXContentListener; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.Locale; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import static org.opensearch.action.support.IndicesOptions.strictExpandOpen; | ||
| import static org.opensearch.knn.common.KNNConstants.CLEAR_CACHE; | ||
| import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; | ||
|
|
||
| /** | ||
| * RestHandler for k-NN Clear Cache API. API provides the ability for a user to evict those indices from Cache. | ||
| */ | ||
| @AllArgsConstructor | ||
| @Log4j2 | ||
| public class RestClearCacheHandler extends BaseRestHandler { | ||
| private static final String INDEX = "index"; | ||
| public static String NAME = "knn_clear_cache_action"; | ||
| private final ClusterService clusterService; | ||
| private final IndexNameExpressionResolver indexNameExpressionResolver; | ||
|
|
||
| /** | ||
| * @return name of Clear Cache API action | ||
| */ | ||
| @Override | ||
| public String getName() { | ||
| return NAME; | ||
| } | ||
|
|
||
| /** | ||
| * @return Immutable List of Clear Cache API endpoint | ||
| */ | ||
| @Override | ||
| public List<Route> routes() { | ||
| return ImmutableList.of( | ||
| new Route(RestRequest.Method.POST, String.format(Locale.ROOT, "%s/%s/{%s}", KNNPlugin.KNN_BASE_URI, CLEAR_CACHE, INDEX)) | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * @param request RestRequest | ||
| * @param client NodeClient | ||
| * @return RestChannelConsumer | ||
| */ | ||
| @Override | ||
| protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { | ||
| ClearCacheRequest clearCacheRequest = createClearCacheRequest(request); | ||
| log.info("[KNN] ClearCache started for the following indices: [{}]", String.join(",", clearCacheRequest.indices())); | ||
| return channel -> client.execute(ClearCacheAction.INSTANCE, clearCacheRequest, new RestToXContentListener<>(channel)); | ||
| } | ||
|
|
||
| // Create a clear cache request by processing the rest request and validating the indices | ||
| private ClearCacheRequest createClearCacheRequest(RestRequest request) { | ||
| String[] indexNames = Strings.splitStringByCommaToArray(request.param("index")); | ||
| Index[] indices = indexNameExpressionResolver.concreteIndices(clusterService.state(), strictExpandOpen(), indexNames); | ||
| validateIndices(indices); | ||
|
|
||
| return new ClearCacheRequest(indexNames); | ||
| } | ||
|
|
||
| // Validate if the given indices are k-NN indices or not. If there are any invalid indices, | ||
| // the request is rejected and an exception is thrown. | ||
| private void validateIndices(Index[] indices) { | ||
| List<String> invalidIndexNames = Arrays.stream(indices) | ||
| .filter(index -> !"true".equals(clusterService.state().metadata().getIndexSafe(index).getSettings().get(KNN_INDEX))) | ||
| .map(Index::getName) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| if (!invalidIndexNames.isEmpty()) { | ||
| throw new KNNInvalidIndicesException( | ||
| invalidIndexNames, | ||
| "ClearCache request rejected. One or more indices have 'index.knn' set to false." | ||
| ); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.