Adding test runner based on docker

Signed-off-by: Martin Gaievski <[email protected]>
opensearch-project · Apr 7, 2023 · 8c6ac66 · 8c6ac66
1 parent 1b6fd48
commit 8c6ac66
Show file tree

Hide file tree

Showing 2 changed files with 97 additions and 0 deletions.
diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -0,0 +1,96 @@
+name: Security test workflow for k-NN
+on:
+  push:
+    branches:
+      - "*"
+      - "feature/**"
+  pull_request:
+    branches:
+      - "*"
+      - "feature/**"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [ 11,17 ]
+        os: [ubuntu-latest]
+      fail-fast: false
+
+    name: Security test workflow for k-NN Plugin
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout k-NN
+        uses: actions/checkout@v1
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Install dependencies on ubuntu
+        if: startsWith(matrix.os,'ubuntu')
+        run: |
+          sudo apt-get install libopenblas-dev gfortran -y
+
+      - name: Assemble k-NN
+        run: |
+          ./gradlew assemble
+      # example of variables:
+      # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip
+      # version = 2.7.0
+      # plugin_version = 2.7.0.0
+      # qualifier = `SNAPSHOT`
+      - name: Pull and Run Docker
+        run: |
+          plugin=`basename $(ls build/distributions/*.zip)`
+          version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
+          plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
+          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`  
+          if [ $qualifier != `SNAPSHOT` ];
+           then
+            docker_version=$version-$qualifier
+          else
+            docker_version=$version
+          fi
+          echo plugin version plugin_version qualifier docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
+               
+          cd ..
+          if docker pull opensearchstaging/opensearch:$docker_version
+          then
+            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
+            # knn plugin cannot be deleted until there are plugin that has dependency on it
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile
+            # saving pre-built artifacts of native libraries as we can't build it with gradle assemle
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile
+            echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
+            # moving pre-built artifacts of native libraries back to plugin folder
+            echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile
+            docker build -t opensearch-knn:test .
+            echo "imagePresent=true" >> $GITHUB_ENV          
+          else
+            echo "imagePresent=false" >> $GITHUB_ENV
+          fi
+
+      - name: Run Docker Image
+        if: env.imagePresent == 'true'
+        run: |
+          cd ..
+          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test
+          sleep 90
+      - name: Run k-NN Integ Test
+        if: env.imagePresent == 'true'
+        run: |
+          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
+          if [ $security -gt 0 ]
+          then
+            echo "Security plugin is available"
+            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
+          else
+            echo "Security plugin is NOT available, skipping integration tests"
+          fi
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827))
 * Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844))
 * Add support of .opensearch-knn-model as system index to transport actions ([#847](https://github.com/opensearch-project/k-NN/pull/847))
+* Add github action for secure integ tests ([#836](https://github.com/opensearch-project/k-NN/pull/836))
 ### Documentation
 ### Maintenance
 ### Refactoring