Add patch to support multi vector in faiss #113
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test k-NN | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every night | |
| push: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| pull_request: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| jobs: | |
| Get-CI-Image-Tag: | |
| uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main | |
| with: | |
| product: opensearch | |
| Build-k-NN-Linux: | |
| strategy: | |
| matrix: | |
| java: [11, 17, 21] | |
| name: Build and Test k-NN Plugin on Linux | |
| runs-on: ubuntu-latest | |
| needs: Get-CI-Image-Tag | |
| container: | |
| # using the same image which is used by opensearch-build team to build the OpenSearch Distribution | |
| # this image tag is subject to change as more dependencies and updates will arrive over time | |
| image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} | |
| # need to switch to root so that github actions can install runner binary on container without permission issues. | |
| options: --user root | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| with: | |
| submodules: true | |
| # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. | |
| # Deleting file at the end to skip `git apply` inside CMAKE file | |
| - name: Apply Git Patch | |
| run: | | |
| cd jni/external/faiss | |
| git config --global core.autocrlf input | |
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch | |
| rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch | |
| working-directory: ${{ github.workspace }} | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Run build | |
| # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. | |
| run: | | |
| chown -R 1000:1000 `pwd` | |
| su `id -un 1000` -c "whoami && java -version && ./gradlew build" | |
| - name: Upload Coverage Report | |
| uses: codecov/codecov-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| Build-k-NN-MacOS: | |
| strategy: | |
| matrix: | |
| java: [ 11, 17, 21 ] | |
| name: Build and Test k-NN Plugin on MacOS | |
| needs: Get-CI-Image-Tag | |
| runs-on: macos-latest | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Install dependencies on macos | |
| run: | | |
| brew reinstall gcc | |
| export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran | |
| - name: Run build | |
| run: | | |
| ./gradlew build | |
| Build-k-NN-Windows: | |
| strategy: | |
| matrix: | |
| java: [ 11, 17, 21 ] | |
| name: Build and Test k-NN Plugin on Windows | |
| needs: Get-CI-Image-Tag | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Install MinGW Using Scoop | |
| run: | | |
| iex "& {$(irm get.scoop.sh)} -RunAsAdmin" | |
| scoop bucket add main | |
| scoop install mingw | |
| - name: Add MinGW to PATH | |
| run: | | |
| echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH | |
| Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" | |
| refreshenv | |
| - name: Download OpenBLAS | |
| run: | | |
| curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip | |
| mkdir OpenBLAS | |
| Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ | |
| mkdir ./src/main/resources/windowsDependencies | |
| cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ | |
| rm .\OpenBLAS-0.3.21-x64.zip | |
| rm -r .\OpenBLAS\ | |
| - name: Run build | |
| run: | | |
| ./gradlew.bat build | |
| # - name: Pull and Run Docker for security tests | |
| # run: | | |
| # plugin=`ls build/distributions/*.zip` | |
| # version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` | |
| # plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` | |
| # echo $version | |
| # cd .. | |
| # if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version | |
| # then | |
| # echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile | |
| # echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile | |
| # echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile | |
| # echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile | |
| # echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile | |
| # echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile | |
| # echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile | |
| # echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile | |
| # echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile | |
| # docker build -t odfe-knn:test . | |
| # echo "imagePresent=true" >> $GITHUB_ENV | |
| # else | |
| # echo "imagePresent=false" >> $GITHUB_ENV | |
| # fi | |
| # - name: Run Docker Image | |
| # if: env.imagePresent == 'true' | |
| # run: | | |
| # cd .. | |
| # docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test | |
| # sleep 90 | |
| # - name: Run k-NN Test | |
| # if: env.imagePresent == 'true' | |
| # run: | | |
| # security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l` | |
| # if [ $security -gt 0 ] | |
| # then | |
| # echo "Security plugin is available. Running tests in security mode" | |
| # ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin | |
| # else | |
| # echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build" | |
| # fi |