Merge branch 'main' into segrep-tests

Signed-off-by: Suraj Singh <[email protected]>

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski
+*   @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski @ryanbogan

.github/workflows/CI.yml

@@ -12,31 +12,76 @@ on:
       - "feature/**"
 
 jobs:
-  Build-k-NN:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  Build-k-NN-Linux:
     strategy:
       matrix:
-        java: [11, 17]
-        os: [ubuntu-latest, macos-latest]
-
-    name: Build and Test k-NN Plugin
-    runs-on: ${{ matrix.os }}
+        java: [11, 17, 21]
+
+    name: Build and Test k-NN Plugin on Linux
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v1
+        with:
+          submodules: true
+
+      # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here.
+      - name: Apply Git Patch
+        # Deleting file at the end to skip `git apply` inside CMAKE file
+        run: |
+          cd jni/external/faiss
+          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
+          rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
+        working-directory: ${{ github.workspace }}
 
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies on ubuntu
-        if: startsWith(matrix.os,'ubuntu')
+      - name: Run build
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
-          sudo apt-get install libopenblas-dev gfortran -y
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew build"
+
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  Build-k-NN-MacOS:
+    strategy:
+      matrix:
+        java: [ 11, 17, 21 ]
+
+    name: Build and Test k-NN Plugin on MacOS
+    needs: Get-CI-Image-Tag
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout k-NN
+        uses: actions/checkout@v1
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
 
       - name: Install dependencies on macos
-        if: startsWith(matrix.os, 'macos')
         run: |
           brew reinstall gcc
           export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran
@@ -45,18 +90,13 @@ jobs:
         run: |
           ./gradlew build
 
-      - name: Upload Coverage Report
-        if: startsWith(matrix.os,'ubuntu')
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   Build-k-NN-Windows:
     strategy:
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
 
     name: Build and Test k-NN Plugin on Windows
+    needs: Get-CI-Image-Tag
     runs-on: windows-latest
 
     steps:
@@ -77,6 +117,7 @@ jobs:
       - name: Add MinGW to PATH
         run: |
           echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
+          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
           refreshenv
 
       - name: Download OpenBLAS
@@ -93,43 +134,3 @@ jobs:
         run: |
           ./gradlew.bat build
 
-#      - name: Pull and Run Docker for security tests
-#        run: |
-#          plugin=`ls build/distributions/*.zip`
-#          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
-#          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
-#          echo $version
-#          cd ..
-#          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version
-#          then
-#            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile
-#            echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile
-#            echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile
-#            echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile
-#            echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile
-#            echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile
-#            echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile
-#            echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile
-#            echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile
-#            docker build -t odfe-knn:test .
-#            echo "imagePresent=true" >> $GITHUB_ENV
-#          else
-#            echo "imagePresent=false" >> $GITHUB_ENV
-#          fi
-#      - name: Run Docker Image
-#        if: env.imagePresent == 'true'
-#        run: |
-#          cd ..
-#          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test
-#          sleep 90
-#      - name: Run k-NN Test
-#        if: env.imagePresent == 'true'
-#        run: |
-#          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l`
-#          if [ $security -gt 0 ]
-#          then
-#            echo "Security plugin is available. Running tests in security mode"
-#            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-#          else
-#            echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build"
-#          fi

.github/workflows/auto-release.yml

@@ -0,0 +1,28 @@
+name: Releases
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+      - uses: actions/checkout@v2
+      - uses: ncipollo/release-action@v1
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          bodyFile: release-notes/opensearch-knn.release-notes-${{steps.tag.outputs.tag}}.md

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         java: [ 11, 17 ]
         os: [ubuntu-latest]
-        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0-SNAPSHOT" ]
+        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0-SNAPSHOT"]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
         exclude:
           - os: windows-latest
@@ -88,7 +88,7 @@ jobs:
       matrix:
         java: [ 11, 17 ]
         os: [ubuntu-latest]
-        bwc_version: [ "2.8.0-SNAPSHOT" ]
+        bwc_version: [ "2.12.0-SNAPSHOT" ]
         opensearch_version: [ "3.0.0-SNAPSHOT" ]
 
     name: k-NN Rolling-Upgrade BWC Tests

.github/workflows/test_security.yml

@@ -12,87 +12,48 @@ on:
       - "feature/**"
 
 jobs:
-  Build-ad:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  integ-test-with-security-linux:
     strategy:
       matrix:
-        java: [ 11,17 ]
-        os: [ubuntu-latest]
-      fail-fast: true
+        java: [11, 17, 21]
 
-    name: Test k-NN on Secure Cluster
-    runs-on: ${{ matrix.os }}
+    name: Run Integration Tests on Linux
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v1
+        with:
+          submodules: true
+
+        # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here.
+      - name: Apply Git Patch
+        # Deleting file at the end to skip `git apply` inside CMAKE file
+        run: |
+          cd jni/external/faiss
+          git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
+          rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
+        working-directory: ${{ github.workspace }}
 
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies on ubuntu
-        if: startsWith(matrix.os,'ubuntu')
-        run: |
-          sudo apt-get install libopenblas-dev gfortran -y
-
-      - name: Assemble k-NN
-        run: |
-          ./gradlew assemble
-      # example of variables:
-      # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip
-      # version = 2.7.0
-      # plugin_version = 2.7.0.0
-      # qualifier = `SNAPSHOT`
-      - name: Pull and Run Docker
-        run: |
-          plugin=`basename $(ls build/distributions/*.zip)`
-          version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
-          plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
-          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`  
-          if [ $qualifier != `SNAPSHOT` ];
-           then
-            docker_version=$version-$qualifier
-          else
-            docker_version=$version
-          fi
-          echo plugin version plugin_version qualifier docker_version
-          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
-               
-          cd ..
-          if docker pull opensearchstaging/opensearch:$docker_version
-          then
-            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
-            # knn plugin cannot be deleted until there are plugin that has dependency on it
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile
-            # saving pre-built artifacts of native libraries as we can't build it with gradle assemle
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile
-            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile
-            echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile
-            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
-            # moving pre-built artifacts of native libraries back to plugin folder
-            echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile
-            docker build -t opensearch-knn:test .
-            echo "imagePresent=true" >> $GITHUB_ENV          
-          else
-            echo "imagePresent=false" >> $GITHUB_ENV
-          fi
-
-      - name: Run Docker Image
-        if: env.imagePresent == 'true'
-        run: |
-          cd ..
-          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test
-          sleep 90
-      - name: Run k-NN Integ Test
-        if: env.imagePresent == 'true'
+      - name: Run build
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
-          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
-          if [ $security -gt 0 ]
-          then
-            echo "Security plugin is available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-          else
-            echo "Security plugin is NOT available, skipping integration tests"
-          fi
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"

CHANGELOG.md

@@ -8,17 +8,36 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Enhancements
 ### Bug Fixes
 ### Infrastructure
-* Fix build errors after refactoring of Strings class in core ([#904](https://github.com/opensearch-project/k-NN/pull/904))
 ### Documentation
 ### Maintenance
 ### Refactoring
 
-## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.7...2.x)
+## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.12...2.x)
 ### Features
+* Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182)
+* Add parent join support for faiss hnsw [#1398](https://github.com/opensearch-project/k-NN/pull/1398)
 ### Enhancements
-* Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773))
+* Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346)
+* Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353)
+* Enabled Filtering on Nested Vector fields with top level filters [#1372](https://github.com/opensearch-project/k-NN/pull/1372)
+* Throw proper exception to invalid k-NN query [#1380](https://github.com/opensearch-project/k-NN/pull/1380)
 ### Bug Fixes
+* Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305)
+* Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318)
+* Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317)
+* Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367)
+* Fix KNNScorer to apply boost [#1403](https://github.com/opensearch-project/k-NN/pull/1403)
+* Fix equals and hashCode methods for KNNQuery and KNNQueryBuilder [#1397](https://github.com/opensearch-project/k-NN/pull/1397)
 ### Infrastructure
+* Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289)
+* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307)
+* Refactor integ tests that access model index [#1423](https://github.com/opensearch-project/k-NN/pull/1423)
+* Fix flaky model tests [#1429](https://github.com/opensearch-project/k-NN/pull/1429)
 ### Documentation
 ### Maintenance
-### Refactoring
+* Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222)
+* Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278)
+* Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319)
+* Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319)
+* Bump lucene codec to 99 [#1383](https://github.com/opensearch-project/k-NN/pull/1383)
+### Refactoring