Further improvements to getting security working

Signed-off-by: John Mazanec <[email protected]>

.github/workflows/CI.yml

@@ -125,43 +125,3 @@ jobs:
         run: |
           ./gradlew.bat build
 
-#      - name: Pull and Run Docker for security tests
-#        run: |
-#          plugin=`ls build/distributions/*.zip`
-#          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
-#          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
-#          echo $version
-#          cd ..
-#          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version
-#          then
-#            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile
-#            echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile
-#            echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile
-#            echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile
-#            echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile
-#            echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile
-#            echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile
-#            echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile
-#            echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile
-#            docker build -t odfe-knn:test .
-#            echo "imagePresent=true" >> $GITHUB_ENV
-#          else
-#            echo "imagePresent=false" >> $GITHUB_ENV
-#          fi
-#      - name: Run Docker Image
-#        if: env.imagePresent == 'true'
-#        run: |
-#          cd ..
-#          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test
-#          sleep 90
-#      - name: Run k-NN Test
-#        if: env.imagePresent == 'true'
-#        run: |
-#          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l`
-#          if [ $security -gt 0 ]
-#          then
-#            echo "Security plugin is available. Running tests in security mode"
-#            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-#          else
-#            echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build"
-#          fi

.github/workflows/test_security.yml

@@ -12,87 +12,39 @@ on:
       - "feature/**"
 
 jobs:
-  Build-ad:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  integ-test-with-security-linux:
     strategy:
       matrix:
-        java: [ 11,17,21 ]
-        os: [ubuntu-latest]
-      fail-fast: true
+        java: [11, 17, 21]
 
-    name: Test k-NN on Secure Cluster
-    runs-on: ${{ matrix.os }}
+    name: Build and Test k-NN Plugin on Linux
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v1
+        with:
+          submodules: true
 
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies on ubuntu
-        if: startsWith(matrix.os,'ubuntu')
-        run: |
-          sudo apt-get install libopenblas-dev gfortran -y
-
-      - name: Assemble k-NN
-        run: |
-          ./gradlew assemble
-      # example of variables:
-      # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip
-      # version = 2.7.0
-      # plugin_version = 2.7.0.0
-      # qualifier = `SNAPSHOT`
-      - name: Pull and Run Docker
-        run: |
-          plugin=`basename $(ls build/distributions/*.zip)`
-          version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
-          plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
-          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`  
-          if [ $qualifier != `SNAPSHOT` ];
-           then
-            docker_version=$version-$qualifier
-          else
-            docker_version=$version
-          fi
-          echo plugin version plugin_version qualifier docker_version
-          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
-               
-          cd ..
-          if docker pull opensearchstaging/opensearch:$docker_version
-          then
-            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
-            # knn plugin cannot be deleted until there are plugin that has dependency on it
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile
-            # saving pre-built artifacts of native libraries as we can't build it with gradle assemle
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile
-            echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
-            # moving pre-built artifacts of native libraries back to plugin folder
-            echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile
-            docker build -t opensearch-knn:test .
-            echo "imagePresent=true" >> $GITHUB_ENV          
-          else
-            echo "imagePresent=false" >> $GITHUB_ENV
-          fi
-
-      - name: Run Docker Image
-        if: env.imagePresent == 'true'
-        run: |
-          cd ..
-          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test
-          sleep 90
-      - name: Run k-NN Integ Test
-        if: env.imagePresent == 'true'
+      - name: Run build
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
-          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
-          if [ $security -gt 0 ]
-          then
-            echo "Security plugin is available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-          else
-            echo "Security plugin is NOT available, skipping integration tests"
-          fi
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"

build.gradle

@@ -4,8 +4,9 @@
  */
 
 import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.testclusters.OpenSearchCluster
 import org.apache.tools.ant.taskdefs.condition.Os
-
+import java.nio.file.Paths
 import java.util.concurrent.Callable
 
 buildscript {
@@ -58,6 +59,7 @@ plugins {
     id 'idea'
     id "com.diffplug.spotless" version "6.20.0" apply false
     id 'io.freefair.lombok' version '8.4'
+    id "de.undercouch.download" version "5.3.0"
 }
 
 apply from: 'gradle/formatting.gradle'
@@ -66,8 +68,66 @@ apply plugin: 'opensearch.rest-test'
 apply plugin: 'opensearch.pluginzip'
 apply plugin: 'opensearch.repositories'
 
+
+def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
+opensearch_tmp_dir.mkdirs()
+
 ext {
     projectSubstitutions = [:]
+
+    configureSecurityPlugin = { OpenSearchCluster cluster ->
+        configurations.zipArchive.asFileTree.each {
+            cluster.plugin(provider(new Callable<RegularFile>() {
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            return it
+                        }
+                    }
+                }
+            }))
+        }
+
+        cluster.getNodes().forEach { node ->
+            var creds = node.getCredentials()
+            if (creds.isEmpty()) {
+                creds.add(Map.of('username', 'admin', 'password', 'admin'))
+            } else {
+                creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
+            }
+        }
+
+        // Config below including files are copied from security demo configuration
+        ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file ->
+            File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
+            download.run {
+                src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
+                dest local
+                overwrite false
+            }
+            cluster.extraConfigFile(file, local)
+        }
+
+        cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+        cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+        cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+        cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+        cluster.setting("plugins.security.ssl.http.enabled", "true")
+        cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+        cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+        cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+        cluster.setting("plugins.security.allow_unsafe_democertificates", "true")
+        cluster.setting("plugins.security.allow_default_init_securityindex", "true")
+        cluster.setting("plugins.security.unsupported.inject_user.enabled", "true")
+
+        cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
+        cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
+        cluster.setting('plugins.security.system_indices.enabled', "true")
+        cluster.setting('plugins.security.system_indices.indices', '[".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]')
+        cluster.setSecure(true)
+    }
 }
 
 allprojects {
@@ -204,10 +264,6 @@ dependencies {
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
 }
 
-
-def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
-opensearch_tmp_dir.mkdirs()
-
 task windowsPatches(type:Exec) {
     commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1"
 }
@@ -252,13 +308,13 @@ integTest {
     // allows integration test classes to access test resource from project root path
     systemProperty('project.root', project.rootDir.absolutePath)
 
-    systemProperty("tests.opensearch.https", "true")
-    systemProperty("tests.opensearch.username", "admin")
-    systemProperty("tests.opensearch.password", "admin")
-
-//    systemProperty "https", System.getProperty("https")
-//    systemProperty "user", System.getProperty("user")
-//    systemProperty "password", System.getProperty("password")
+    if (System.getProperty("security.enabled") != null) {
+        // These need to be set for configuring the rest client
+        systemProperty("https", "true")
+        systemProperty("secure", "true")
+        systemProperty("user", "admin")
+        systemProperty("password", "admin")
+    }
 
     doFirst {
         // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
@@ -269,10 +325,6 @@ integTest {
         systemProperty 'cluster.number_of_nodes', "${_numNodes}"
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
         // not being written, the waitForAllConditions ensures it's written
-
-
-
-
         getClusters().forEach { cluster ->
             cluster.waitForAllConditions()
         }
@@ -289,52 +341,7 @@ testClusters.integTest {
 
     // Optionally install security
     if (System.getProperty("security.enabled") != null) {
-        configurations.zipArchive.asFileTree.each {
-            plugin(provider(new Callable<RegularFile>() {
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            return it
-                        }
-                    }
-                }
-            }))
-        }
-
-//        systemProperty("tests.opensearch.https", "true")
-//        systemProperty("tests.opensearch.username", "admin")
-//        systemProperty("tests.opensearch.password", "admin")
-
-        systemProperty("secure", "true")
-        systemProperty("username", "admin")
-        systemProperty("password", "admin")
-
-        extraConfigFile("admin-cert.pem", new File("$rootDir/src/test/resources/security/admin-cert.pem"))
-        extraConfigFile("node-cert.pem", new File("$rootDir/src/test/resources/security/node-cert.pem"))
-        extraConfigFile("node-key.pem", new File("$rootDir/src/test/resources/security/node-key.pem"))
-        extraConfigFile("root-ca.pem", new File("$rootDir/src/test/resources/security/root-ca.pem"))
-
-        setting("plugins.security.ssl.transport.pemcert_filepath", "node-cert.pem")
-        setting("plugins.security.ssl.transport.pemkey_filepath", "node-key.pem")
-        setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
-        setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
-        setting("plugins.security.ssl.http.enabled", "true")
-        setting("plugins.security.ssl.http.pemcert_filepath", "node-cert.pem")
-        setting("plugins.security.ssl.http.pemkey_filepath", "node-key.pem")
-        setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
-        setting("plugins.security.allow_unsafe_democertificates", "true")
-        setting("plugins.security.allow_default_init_securityindex", "true")
-        //setting("plugins.security.unsupported.inject_user.enabled", "true")
-
-
-        setting("plugins.security.authcz.admin_dn", "\n- CN=admin,OU=client,O=client,L=test, C=de")
-        setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
-        setting('plugins.security.system_indices.enabled', "true")
-        setting('plugins.security.system_indices.indices', '[".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"]')
-
-        setSecure(true)
+        configureSecurityPlugin(testClusters.integTest)
     }
 
     plugin(project.tasks.bundlePlugin.archiveFile)