Install security plugin from individual artifacts

.github/workflows/CI.yml

@@ -125,43 +125,3 @@ jobs:
         run: |
           ./gradlew.bat build
 
-#      - name: Pull and Run Docker for security tests
-#        run: |
-#          plugin=`ls build/distributions/*.zip`
-#          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
-#          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
-#          echo $version
-#          cd ..
-#          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version
-#          then
-#            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile
-#            echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile
-#            echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile
-#            echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile
-#            echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile
-#            echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile
-#            echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile
-#            echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile
-#            echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile
-#            docker build -t odfe-knn:test .
-#            echo "imagePresent=true" >> $GITHUB_ENV
-#          else
-#            echo "imagePresent=false" >> $GITHUB_ENV
-#          fi
-#      - name: Run Docker Image
-#        if: env.imagePresent == 'true'
-#        run: |
-#          cd ..
-#          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test
-#          sleep 90
-#      - name: Run k-NN Test
-#        if: env.imagePresent == 'true'
-#        run: |
-#          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l`
-#          if [ $security -gt 0 ]
-#          then
-#            echo "Security plugin is available. Running tests in security mode"
-#            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-#          else
-#            echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build"
-#          fi

.github/workflows/test_security.yml

@@ -12,87 +12,39 @@ on:
       - "feature/**"
 
 jobs:
-  Build-ad:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  integ-test-with-security-linux:
     strategy:
       matrix:
-        java: [ 11,17,21 ]
-        os: [ubuntu-latest]
-      fail-fast: true
+        java: [11, 17, 21]
 
-    name: Test k-NN on Secure Cluster
-    runs-on: ${{ matrix.os }}
+    name: Run Integration Tests on Linux
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v1
+        with:
+          submodules: true
 
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies on ubuntu
-        if: startsWith(matrix.os,'ubuntu')
-        run: |
-          sudo apt-get install libopenblas-dev gfortran -y
-
-      - name: Assemble k-NN
-        run: |
-          ./gradlew assemble
-      # example of variables:
-      # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip
-      # version = 2.7.0
-      # plugin_version = 2.7.0.0
-      # qualifier = `SNAPSHOT`
-      - name: Pull and Run Docker
-        run: |
-          plugin=`basename $(ls build/distributions/*.zip)`
-          version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
-          plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
-          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`  
-          if [ $qualifier != `SNAPSHOT` ];
-           then
-            docker_version=$version-$qualifier
-          else
-            docker_version=$version
-          fi
-          echo plugin version plugin_version qualifier docker_version
-          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
-
-          cd ..
-          if docker pull opensearchstaging/opensearch:$docker_version
-          then
-            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
-            # knn plugin cannot be deleted until there are plugin that has dependency on it
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile
-            # saving pre-built artifacts of native libraries as we can't build it with gradle assemle
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile
-            echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
-            # moving pre-built artifacts of native libraries back to plugin folder
-            echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile
-            docker build -t opensearch-knn:test .
-            echo "imagePresent=true" >> $GITHUB_ENV          
-          else
-            echo "imagePresent=false" >> $GITHUB_ENV
-          fi
-
-      - name: Run Docker Image
-        if: env.imagePresent == 'true'
-        run: |
-          cd ..
-          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test
-          sleep 90
-      - name: Run k-NN Integ Test
-        if: env.imagePresent == 'true'
+      - name: Run build
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
-          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
-          if [ $security -gt 0 ]
-          then
-            echo "Security plugin is available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-          else
-            echo "Security plugin is NOT available, skipping integration tests"
-          fi
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"

CHANGELOG.md

@@ -22,9 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305)
 * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318)
 * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317)
-
 ### Infrastructure
 * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289)
+* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307)
 ### Documentation
 ### Maintenance
 * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222)

DEVELOPER_GUIDE.md

@@ -264,6 +264,38 @@ curl localhost:9200
   }
 }
 ```
+
+Additionally, it is also possible to run a cluster with security enabled:
+```shell script
+./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin
+```
+
+By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and
+`password=admin`.
+
+Then, to access the cluster, we can run
+```bash
+curl https://localhost:9200 --insecure -u admin:admin
+
+{
+  "name" : "integTest-0",
+  "cluster_name" : "integTest",
+  "cluster_uuid" : "kLsNk4JDTMyp1yQRqog-3g",
+  "version" : {
+    "distribution" : "opensearch",
+    "number" : "3.0.0-SNAPSHOT",
+    "build_type" : "tar",
+    "build_hash" : "9d85e566894ef53e5f2093618b3d455e4d0a04ce",
+    "build_date" : "2023-10-30T18:34:06.996519Z",
+    "build_snapshot" : true,
+    "lucene_version" : "9.8.0",
+    "minimum_wire_compatibility_version" : "2.12.0",
+    "minimum_index_compatibility_version" : "2.0.0"
+  },
+  "tagline" : "The OpenSearch Project: https://opensearch.org/"
+}
+```
+
 ### Run Multi-node Cluster Locally
 
 It can be useful to test and debug on a multi-node cluster. In order to launch a 3 node cluster with the KNN plugin installed, run the following command:
@@ -272,12 +304,17 @@ It can be useful to test and debug on a multi-node cluster. In order to launch a
 ./gradlew run -PnumNodes=3
 ```
 
-In order to run the integration tests with a 3 node cluster, run this command:
+In order to run the integration tests, run this command:
 
 ```
 ./gradlew :integTest -PnumNodes=3
 ```
 
+Additionally, to run integration tests with security enabled, run
+```
+./gradlew :integTest -Dsecurity.enabled=true -PnumNodes=3
+```
+
 Integration tests can be run with remote cluster. For that run the following command and replace host/port/cluster name values with ones for the target cluster:
 
 ```