diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c807e348e6..b58741b921 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -125,43 +125,3 @@ jobs: run: | ./gradlew.bat build -# - name: Pull and Run Docker for security tests -# run: | -# plugin=`ls build/distributions/*.zip` -# version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` -# plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` -# echo $version -# cd .. -# if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version -# then -# echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile -# echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile -# echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile -# echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile -# echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile -# echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile -# echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile -# echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile -# echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile -# docker build -t odfe-knn:test . -# echo "imagePresent=true" >> $GITHUB_ENV -# else -# echo "imagePresent=false" >> $GITHUB_ENV -# fi -# - name: Run Docker Image -# if: env.imagePresent == 'true' -# run: | -# cd .. -# docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test -# sleep 90 -# - name: Run k-NN Test -# if: env.imagePresent == 'true' -# run: | -# security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l` -# if [ $security -gt 0 ] -# then -# echo "Security plugin is available. Running tests in security mode" -# ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin -# else -# echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build" -# fi diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index ff3ca14595..4d0374a081 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -12,87 +12,39 @@ on: - "feature/**" jobs: - Build-ad: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + integ-test-with-security-linux: strategy: matrix: - java: [ 11,17,21 ] - os: [ubuntu-latest] - fail-fast: true + java: [11, 17, 21] - name: Test k-NN on Secure Cluster - runs-on: ${{ matrix.os }} + name: Run Integration Tests on Linux + runs-on: ubuntu-latest + needs: Get-CI-Image-Tag + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Checkout k-NN uses: actions/checkout@v1 + with: + submodules: true - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - name: Install dependencies on ubuntu - if: startsWith(matrix.os,'ubuntu') - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - name: Assemble k-NN - run: | - ./gradlew assemble - # example of variables: - # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip - # version = 2.7.0 - # plugin_version = 2.7.0.0 - # qualifier = `SNAPSHOT` - - name: Pull and Run Docker - run: | - plugin=`basename $(ls build/distributions/*.zip)` - version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` - plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` - qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` - if [ $qualifier != `SNAPSHOT` ]; - then - docker_version=$version-$qualifier - else - docker_version=$version - fi - echo plugin version plugin_version qualifier docker_version - echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" - - cd .. - if docker pull opensearchstaging/opensearch:$docker_version - then - echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile - # knn plugin cannot be deleted until there are plugin that has dependency on it - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile - # saving pre-built artifacts of native libraries as we can't build it with gradle assemle - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile - echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile - echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile - # moving pre-built artifacts of native libraries back to plugin folder - echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile - docker build -t opensearch-knn:test . - echo "imagePresent=true" >> $GITHUB_ENV - else - echo "imagePresent=false" >> $GITHUB_ENV - fi - - - name: Run Docker Image - if: env.imagePresent == 'true' - run: | - cd .. - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test - sleep 90 - - name: Run k-NN Integ Test - if: env.imagePresent == 'true' + - name: Run build + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` - if [ $security -gt 0 ] - then - echo "Security plugin is available" - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin - else - echo "Security plugin is NOT available, skipping integration tests" - fi + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" diff --git a/CHANGELOG.md b/CHANGELOG.md index bf61d49e7b..7839d98254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,9 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) - ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) +* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index cbfaf0f4c6..2e7f7322c8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -264,6 +264,38 @@ curl localhost:9200 } } ``` + +Additionally, it is also possible to run a cluster with security enabled: +```shell script +./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin +``` + +By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and +`password=admin`. + +Then, to access the cluster, we can run +```bash +curl https://localhost:9200 --insecure -u admin:admin + +{ + "name" : "integTest-0", + "cluster_name" : "integTest", + "cluster_uuid" : "kLsNk4JDTMyp1yQRqog-3g", + "version" : { + "distribution" : "opensearch", + "number" : "3.0.0-SNAPSHOT", + "build_type" : "tar", + "build_hash" : "9d85e566894ef53e5f2093618b3d455e4d0a04ce", + "build_date" : "2023-10-30T18:34:06.996519Z", + "build_snapshot" : true, + "lucene_version" : "9.8.0", + "minimum_wire_compatibility_version" : "2.12.0", + "minimum_index_compatibility_version" : "2.0.0" + }, + "tagline" : "The OpenSearch Project: https://opensearch.org/" +} +``` + ### Run Multi-node Cluster Locally It can be useful to test and debug on a multi-node cluster. In order to launch a 3 node cluster with the KNN plugin installed, run the following command: @@ -272,12 +304,17 @@ It can be useful to test and debug on a multi-node cluster. In order to launch a ./gradlew run -PnumNodes=3 ``` -In order to run the integration tests with a 3 node cluster, run this command: +In order to run the integration tests, run this command: ``` ./gradlew :integTest -PnumNodes=3 ``` +Additionally, to run integration tests with security enabled, run +``` +./gradlew :integTest -Dsecurity.enabled=true -PnumNodes=3 +``` + Integration tests can be run with remote cluster. For that run the following command and replace host/port/cluster name values with ones for the target cluster: ``` diff --git a/build.gradle b/build.gradle index ceba9a038f..2ed70a66e8 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,10 @@ */ import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.OpenSearchCluster import org.apache.tools.ant.taskdefs.condition.Os +import java.nio.file.Paths +import java.util.concurrent.Callable buildscript { ext { @@ -13,6 +16,19 @@ buildscript { opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + plugin_no_snapshot = opensearch_build + if (version_qualifier) { + opensearch_build += "-${version_qualifier}" + plugin_no_snapshot += "-${version_qualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } + opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") } // This isn't applying from repositories.gradle so repeating git diff it here @@ -43,6 +59,7 @@ plugins { id 'idea' id "com.diffplug.spotless" version "6.20.0" apply false id 'io.freefair.lombok' version '8.4' + id "de.undercouch.download" version "5.3.0" } apply from: 'gradle/formatting.gradle' @@ -51,9 +68,91 @@ apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' apply plugin: 'opensearch.repositories' + +def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile +opensearch_tmp_dir.mkdirs() + ext { - isSnapshot = "true" == System.getProperty("build.snapshot", "true") projectSubstitutions = [:] + + configureSecurityPlugin = { OpenSearchCluster cluster -> + configurations.zipArchive.asFileTree.each { + cluster.plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + + cluster.getNodes().forEach { node -> + var creds = node.getCredentials() + if (creds.isEmpty()) { + creds.add(Map.of('username', 'admin', 'password', 'admin')) + } else { + creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin')) + } + } + + // Config below including files are copied from security demo configuration + ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file -> + File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + cluster.extraConfigFile(file, local) + } + + // This configuration is copied from the security plugins demo install: + // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388 + cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + cluster.setting("plugins.security.ssl.http.enabled", "true") + cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.allow_unsafe_democertificates", "true") + cluster.setting("plugins.security.allow_default_init_securityindex", "true") + cluster.setting("plugins.security.unsupported.inject_user.enabled", "true") + + cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de") + cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') + cluster.setting('plugins.security.system_indices.enabled', "true") + cluster.setting('plugins.security.system_indices.indices', '[' + + '".plugins-ml-config", ' + + '".plugins-ml-connector", ' + + '".plugins-ml-model-group", ' + + '".plugins-ml-model", ".plugins-ml-task", ' + + '".plugins-ml-conversation-meta", ' + + '".plugins-ml-conversation-interactions", ' + + '".opendistro-alerting-config", ' + + '".opendistro-alerting-alert*", ' + + '".opendistro-anomaly-results*", ' + + '".opendistro-anomaly-detector*", ' + + '".opendistro-anomaly-checkpoints", ' + + '".opendistro-anomaly-detection-state", ' + + '".opendistro-reports-*", ' + + '".opensearch-notifications-*", ' + + '".opensearch-notebooks", ' + + '".opensearch-observability", ' + + '".ql-datasources", ' + + '".opendistro-asynchronous-search-response*", ' + + '".replication-metadata-store", ' + + '".opensearch-knn-models", ' + + '".geospatial-ip2geo-data*"' + + ']' + ) + cluster.setSecure(true) + } } allprojects { @@ -87,6 +186,10 @@ allprojects { } } +configurations { + zipArchive +} + publishing { repositories { maven { @@ -182,11 +285,9 @@ dependencies { testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7' testFixturesImplementation "org.opensearch:common-utils:${version}" -} - -def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile -opensearch_tmp_dir.mkdirs() + zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" +} task windowsPatches(type:Exec) { commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1" @@ -232,9 +333,18 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) - systemProperty "https", System.getProperty("https") - systemProperty "user", System.getProperty("user") - systemProperty "password", System.getProperty("password") + var is_https = System.getProperty("https") + var user = System.getProperty("user") + var password = System.getProperty("password") + if (System.getProperty("security.enabled") != null) { + // If security is enabled, set is_https/user/password defaults + is_https = is_https == null ? "true" : is_https + user = user == null ? "admin" : user + password = password == null ? "admin" : password + } + systemProperty("https", is_https) + systemProperty("user", user) + systemProperty("password", password) doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can @@ -258,6 +368,12 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" + + // Optionally install security + if (System.getProperty("security.enabled") != null) { + configureSecurityPlugin(testClusters.integTest) + } + plugin(project.tasks.bundlePlugin.archiveFile) if (Os.isFamily(Os.FAMILY_WINDOWS)) { // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index 0ef9c4c914..be20d18a2c 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -10,21 +10,16 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; @@ -55,7 +50,7 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int DOC_ID_TEST_MODEL_INDEX = 0; private static int DOC_ID_TEST_MODEL_INDEX_DEFAULT = 0; private static final int DELAY_MILLI_SEC = 1000; - private static final int EXP_NUM_OF_MODELS = 3; + private static final int EXP_NUM_OF_MODELS = 2; private static final int K = 5; private static final int NUM_DOCS = 10; private static final int NUM_DOCS_TEST_MODEL_INDEX = 100; @@ -66,7 +61,6 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int QUERY_COUNT_TEST_MODEL_INDEX_DEFAULT = 0; private static final String TEST_MODEL_ID = "test-model-id"; private static final String TEST_MODEL_ID_DEFAULT = "test-model-id-default"; - private static final String TEST_MODEL_ID_TRAINING = "test-model-id-training"; private static final String MODEL_DESCRIPTION = "Description for train model test"; // KNN model test @@ -139,22 +133,6 @@ public void testKNNModelDefault() throws Exception { } } - // KNN Delete Model test for model in Training State - public void testDeleteTrainingModel() throws Exception { - byte[] testModelBlob = "hello".getBytes(StandardCharsets.UTF_8); - ModelMetadata testModelMetadata = getModelMetadata(); - testModelMetadata.setState(ModelState.TRAINING); - if (isRunningAgainstOldCluster()) { - addModelToSystemIndex(TEST_MODEL_ID_TRAINING, testModelMetadata, testModelBlob); - } else { - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, TEST_MODEL_ID_TRAINING); - Request request = new Request("DELETE", restURI); - - ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); - assertEquals(RestStatus.CONFLICT.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); - } - } - // Delete Models and ".opensearch-knn-models" index to clear cluster metadata @AfterClass public static void wipeAllModels() throws IOException { @@ -255,8 +233,4 @@ public String modelIndexMapping(String fieldName, String modelId) throws IOExcep .endObject() .toString(); } - - private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", "", ""); - } } diff --git a/src/test/resources/security/sample.pem b/src/test/resources/security/sample.pem deleted file mode 100644 index a1fc20a776..0000000000 --- a/src/test/resources/security/sample.pem +++ /dev/null @@ -1,25 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR -MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 -zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N -1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy -vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L -zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= ------END CERTIFICATE----- diff --git a/src/test/resources/security/test-kirk.jks b/src/test/resources/security/test-kirk.jks deleted file mode 100644 index 6dbc51e714..0000000000 Binary files a/src/test/resources/security/test-kirk.jks and /dev/null differ diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 70605b5993..c2a8a1010a 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -23,7 +23,6 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; @@ -57,7 +56,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -77,15 +75,10 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; -import static org.opensearch.knn.common.KNNConstants.MODEL_NODE_ASSIGNMENT; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; -import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; @@ -737,33 +730,6 @@ protected void createModelSystemIndex() throws IOException { } } - protected void addModelToSystemIndex(String modelId, ModelMetadata modelMetadata, byte[] model) throws IOException { - assertFalse(StringUtils.isBlank(modelId)); - String modelBase64 = Base64.getEncoder().encodeToString(model); - - Request request = new Request("POST", "/" + MODEL_INDEX_NAME + "/_doc/" + modelId + "?refresh=true"); - - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .field(MODEL_ID, modelId) - .field(MODEL_STATE, modelMetadata.getState().getName()) - .field(KNN_ENGINE, modelMetadata.getKnnEngine().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, modelMetadata.getSpaceType().getValue()) - .field(DIMENSION, modelMetadata.getDimension()) - .field(MODEL_BLOB_PARAMETER, modelBase64) - .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp()) - .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) - .field(MODEL_ERROR, modelMetadata.getError()) - .field(MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()) - .endObject(); - - request.setJsonEntity(builder.toString()); - - Response response = client().performRequest(request); - - assertEquals(request.getEndpoint() + ": failed", RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - } - /** * Clear cache *

diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index 230c1e7dc9..9867377e15 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -21,41 +21,62 @@ import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.commons.rest.SecureRestClientBuilder; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_ENABLED; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.search.SearchHit; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.junit.After; + import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; import static org.opensearch.knn.TestUtils.OPENDISTRO_SECURITY; import static org.opensearch.knn.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; @@ -72,15 +93,7 @@ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { private final Set IMMUTABLE_INDEX_PREFIXES = Set.of(KNN_BWC_PREFIX, SECURITY_AUDITLOG_PREFIX, OPENSEARCH_SYSTEM_INDEX_PREFIX); protected boolean isHttps() { - boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); - if (isHttps) { - // currently only external cluster is supported for security enabled testing - if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { - throw new RuntimeException("cluster url should be provided for security enabled testing"); - } - } - - return isHttps; + return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); } @Override @@ -92,37 +105,19 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { - String keystore = settings.get(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH); - if (Objects.nonNull(keystore)) { - URI uri; - try { - uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); - return new SecureRestClientBuilder(settings, configPath).build(); - } else { - configureHttpsClient(builder, settings); - boolean strictDeprecationMode = settings.getAsBoolean("strictDeprecationMode", true); - builder.setStrictDeprecationMode(strictDeprecationMode); - return builder.build(); - } + configureHttpsClient(builder, settings); } else { configureClient(builder, settings); } + builder.setStrictDeprecationMode(false); return builder.build(); } protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { - Map headers = ThreadContext.buildDefaultHeaders(settings); - Header[] defaultHeaders = new Header[headers.size()]; - int i = 0; - for (Map.Entry entry : headers.entrySet()) { - defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); - } - builder.setDefaultHeaders(defaultHeaders); + // Similar to client configuration with OpenSearch: + // https://github.com/opensearch-project/OpenSearch/blob/2.11.1/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java#L841-L863 + // except we set the user name and password builder.setHttpClientConfigCallback(httpClientBuilder -> { String userName = Optional.ofNullable(System.getProperty("user")) .orElseThrow(() -> new RuntimeException("user name is missing")); @@ -131,21 +126,35 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); try { - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - // disable the certificate since our testing cluster just uses the default security configuration - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(sslEngine -> new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol())) + .build(); + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); } catch (Exception e) { throw new RuntimeException(e); } }); - + Map headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); final TimeValue socketTimeout = TimeValue.parseTimeValue( socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT ); - builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + builder.setRequestConfigCallback( + conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()))) + ); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); } @@ -182,8 +191,10 @@ protected void wipeAllODFEIndices() throws Exception { for (Map index : parserList) { final String indexName = (String) index.get("index"); - if (isIndexCleanupRequired(indexName)) { - wipeIndexContent(indexName); + if (MODEL_INDEX_NAME.equals(indexName)) { + if (!getSkipDeleteModelIndexFlag()) { + deleteModels(getModelIds()); + } continue; } if (!skipDeleteIndex(indexName)) { @@ -193,16 +204,7 @@ protected void wipeAllODFEIndices() throws Exception { } } - private boolean isIndexCleanupRequired(final String index) { - return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag(); - } - - private void wipeIndexContent(String indexName) throws IOException { - deleteModels(getModelIds()); - deleteAllDocs(indexName); - } - - private List getModelIds() throws IOException { + private List getModelIds() throws IOException, ParseException { final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); final Response response = adminClient().performRequest(new Request("GET", restURIGetModels)); @@ -229,51 +231,14 @@ private void deleteModels(final List modelIds) throws IOException { } } - private void deleteAllDocs(final String indexName) throws IOException { - final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); - final Request request = new Request("POST", restURIDeleteByQuery); - final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("match_all") - .endObject() - .endObject() - .endObject(); - - request.setJsonEntity(matchAllDocsQuery.toString()); - adminClient().performRequest(request); - } - private boolean getSkipDeleteModelIndexFlag() { return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); } - private boolean skipDeleteModelIndex(String indexName) { - return (MODEL_INDEX_NAME.equals(indexName) && getSkipDeleteModelIndexFlag()); - } - private boolean skipDeleteIndex(String indexName) { - if (indexName != null - && !OPENDISTRO_SECURITY.equals(indexName) - && IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith) - && !skipDeleteModelIndex(indexName)) { - return false; - } - - return true; - } - - @Override - protected Settings restAdminSettings() { - return Settings.builder() - // disable the warning exception for admin client since it's only used for cleanup. - .put("strictDeprecationMode", false) - .put("http.port", 9200) - .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) - .put(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit") - .build(); + return indexName == null + || OPENDISTRO_SECURITY.equals(indexName) + || IMMUTABLE_INDEX_PREFIXES.stream().anyMatch(indexName::startsWith) + || MODEL_INDEX_NAME.equals(indexName); } }