Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add Clear Cache API Signed-off-by: Naveen Tatikonda <[email protected]> * Add Unit and Integration tests Signed-off-by: Naveen Tatikonda <[email protected]> * Add BWC Tests Signed-off-by: Naveen Tatikonda <[email protected]> * Add CHANGELOG Signed-off-by: Naveen Tatikonda <[email protected]> * Address Review Comments Signed-off-by: Naveen Tatikonda <[email protected]> --------- Signed-off-by: Naveen Tatikonda <[email protected]> (cherry picked from commit 12f4a51)
- Loading branch information
1 parent
8c7cb1b
commit 70bca5f
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.bwc; | ||
|
|
||
| import java.util.Collections; | ||
| import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; | ||
|
|
||
| public class ClearCacheIT extends AbstractRestartUpgradeTestCase { | ||
| private static final String TEST_FIELD = "test-field"; | ||
| private static final int DIMENSIONS = 5; | ||
| private static int docId = 0; | ||
| private static final int NUM_DOCS = 10; | ||
| private static int queryCnt = 0; | ||
| private static final int K = 5; | ||
|
|
||
| // Restart Upgrade BWC Tests to validate Clear Cache API | ||
| public void testClearCache() throws Exception { | ||
| waitForClusterHealthGreen(NODES_BWC_CLUSTER); | ||
| if (isRunningAgainstOldCluster()) { | ||
| createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); | ||
| } else { | ||
| queryCnt = NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
|
|
||
| docId = NUM_DOCS; | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); | ||
|
|
||
| queryCnt = queryCnt + NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
| deleteKNNIndex(testIndex); | ||
| } | ||
| } | ||
|
|
||
| // validation steps for Clear Cache API after upgrading node to new version | ||
| private void validateClearCacheOnUpgrade(int queryCount) throws Exception { | ||
| int graphCount = getTotalGraphsInCache(); | ||
| knnWarmup(Collections.singletonList(testIndex)); | ||
| assertTrue(getTotalGraphsInCache() > graphCount); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
|
|
||
| clearCache(Collections.singletonList(testIndex)); | ||
| assertEquals(0, getTotalGraphsInCache()); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.bwc; | ||
|
|
||
| import java.util.Collections; | ||
|
|
||
| import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; | ||
|
|
||
| public class ClearCacheIT extends AbstractRollingUpgradeTestCase { | ||
| private static final String TEST_FIELD = "test-field"; | ||
| private static final int DIMENSIONS = 5; | ||
| private static int docId = 0; | ||
| private static final int K = 5; | ||
| private static final int NUM_DOCS = 10; | ||
| private static int queryCnt = 0; | ||
|
|
||
| // Rolling Upgrade BWC Tests to validate Clear Cache API | ||
| public void testClearCache() throws Exception { | ||
| waitForClusterHealthGreen(NODES_BWC_CLUSTER); | ||
| switch (getClusterType()) { | ||
| case OLD: | ||
| createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); | ||
| int docIdOld = 0; | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); | ||
| break; | ||
| case UPGRADED: | ||
| queryCnt = NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
|
|
||
| docId = NUM_DOCS; | ||
| addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); | ||
|
|
||
| queryCnt = queryCnt + NUM_DOCS; | ||
| validateClearCacheOnUpgrade(queryCnt); | ||
| deleteKNNIndex(testIndex); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| // validation steps for Clear Cache API after upgrading all nodes from old version to new version | ||
| public void validateClearCacheOnUpgrade(int queryCount) throws Exception { | ||
| int graphCount = getTotalGraphsInCache(); | ||
| knnWarmup(Collections.singletonList(testIndex)); | ||
| assertTrue(getTotalGraphsInCache() > graphCount); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
|
|
||
| clearCache(Collections.singletonList(testIndex)); | ||
| assertEquals(0, getTotalGraphsInCache()); | ||
| validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.plugin.rest; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.extern.log4j.Log4j2; | ||
| import org.opensearch.client.node.NodeClient; | ||
| import org.opensearch.cluster.metadata.IndexNameExpressionResolver; | ||
| import org.opensearch.cluster.service.ClusterService; | ||
| import org.opensearch.core.common.Strings; | ||
| import org.opensearch.core.index.Index; | ||
| import org.opensearch.knn.common.exception.KNNInvalidIndicesException; | ||
| import org.opensearch.knn.plugin.KNNPlugin; | ||
| import org.opensearch.knn.plugin.transport.ClearCacheAction; | ||
| import org.opensearch.knn.plugin.transport.ClearCacheRequest; | ||
| import org.opensearch.rest.BaseRestHandler; | ||
| import org.opensearch.rest.RestRequest; | ||
| import org.opensearch.rest.action.RestToXContentListener; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.Locale; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import static org.opensearch.action.support.IndicesOptions.strictExpandOpen; | ||
| import static org.opensearch.knn.common.KNNConstants.CLEAR_CACHE; | ||
| import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; | ||
|
|
||
| /** | ||
| * RestHandler for k-NN Clear Cache API. API provides the ability for a user to evict those indices from Cache. | ||
| */ | ||
| @AllArgsConstructor | ||
| @Log4j2 | ||
| public class RestClearCacheHandler extends BaseRestHandler { | ||
| private static final String INDEX = "index"; | ||
| public static String NAME = "knn_clear_cache_action"; | ||
| private final ClusterService clusterService; | ||
| private final IndexNameExpressionResolver indexNameExpressionResolver; | ||
|
|
||
| /** | ||
| * @return name of Clear Cache API action | ||
| */ | ||
| @Override | ||
| public String getName() { | ||
| return NAME; | ||
| } | ||
|
|
||
| /** | ||
| * @return Immutable List of Clear Cache API endpoint | ||
| */ | ||
| @Override | ||
| public List<Route> routes() { | ||
| return ImmutableList.of( | ||
| new Route(RestRequest.Method.POST, String.format(Locale.ROOT, "%s/%s/{%s}", KNNPlugin.KNN_BASE_URI, CLEAR_CACHE, INDEX)) | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * @param request RestRequest | ||
| * @param client NodeClient | ||
| * @return RestChannelConsumer | ||
| */ | ||
| @Override | ||
| protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { | ||
| ClearCacheRequest clearCacheRequest = createClearCacheRequest(request); | ||
| log.info("[KNN] ClearCache started for the following indices: [{}]", String.join(",", clearCacheRequest.indices())); | ||
| return channel -> client.execute(ClearCacheAction.INSTANCE, clearCacheRequest, new RestToXContentListener<>(channel)); | ||
| } | ||
|
|
||
| // Create a clear cache request by processing the rest request and validating the indices | ||
| private ClearCacheRequest createClearCacheRequest(RestRequest request) { | ||
| String[] indexNames = Strings.splitStringByCommaToArray(request.param("index")); | ||
| Index[] indices = indexNameExpressionResolver.concreteIndices(clusterService.state(), strictExpandOpen(), indexNames); | ||
| validateIndices(indices); | ||
|
|
||
| return new ClearCacheRequest(indexNames); | ||
| } | ||
|
|
||
| // Validate if the given indices are k-NN indices or not. If there are any invalid indices, | ||
| // the request is rejected and an exception is thrown. | ||
| private void validateIndices(Index[] indices) { | ||
| List<String> invalidIndexNames = Arrays.stream(indices) | ||
| .filter(index -> !"true".equals(clusterService.state().metadata().getIndexSafe(index).getSettings().get(KNN_INDEX))) | ||
| .map(Index::getName) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| if (!invalidIndexNames.isEmpty()) { | ||
| throw new KNNInvalidIndicesException( | ||
| invalidIndexNames, | ||
| "ClearCache request rejected. One or more indices have 'index.knn' set to false." | ||
| ); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.