Update to version v5.0.0

Restream · Mar 4, 2025 · 03771e3 · 03771e3
1 parent 3817edd
commit 03771e3
Show file tree

Hide file tree

Showing 1,143 changed files with 174,944 additions and 34,993 deletions.
diff --git a/.github/workflows/install_grpc.sh b/.github/workflows/install_grpc.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-git clone --branch v1.44.0 https://github.com/grpc/grpc
+git clone --branch v1.54.3 https://github.com/grpc/grpc
 cd grpc
 git submodule update --init
 mkdir -p cmake/build

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -6,39 +6,35 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        compiler: ["Visual Studio 17 2022"]
+        compiler: ["Visual Studio 17 2022", "MinGW Makefiles"]
       fail-fast: false
+    env:
+      EXTRA_FLAGS: "${{ matrix.compiler == 'Visual Studio 17 2022' && '-Ax64' || '' }}"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Build Reindexer
         run: |
           mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" .. -Ax64
+          cmake -G "${{matrix.compiler}}" -DBUILD_ANN_INDEXES=builtin .. $EXTRA_FLAGS
           cmake --build . --config Release
           cmake --build . --config Release --target face
           cmake --build . --config Release --target swagger
           cpack
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: windows-latest
-          path: build/*.exe
-          if-no-files-found: error
 
   build-windows-2019:
     runs-on: windows-2019
     strategy:
       matrix:
-        compiler: ["Visual Studio 16 2019", "MinGW Makefiles"]
+        compiler: ["Visual Studio 16 2019"]
       fail-fast: false
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Build Reindexer
         run: |
           mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" ..
+          cmake -G "${{matrix.compiler}}" -DBUILD_ANN_INDEXES=none ..
           cmake --build . --config Release
           cmake --build . --config Release --target face
           cmake --build . --config Release --target swagger
@@ -47,15 +43,10 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
         include:
-          - os: ubuntu-24.04
-            cc: gcc-12
-            cxx: g++-12
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
           - os: ubuntu-latest
             sanitizer: TSAN
             cc: gcc-12
@@ -66,17 +57,18 @@ jobs:
       OS: ${{matrix.os}}
       CC: ${{matrix.cc}}
       CXX: ${{matrix.cxx}}
+      SANITIZER: ${{matrix.sanitizer}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Prepare Environment
         run: |
           ./.github/workflows/install_gtest.sh
-          ./.github/workflows/install_gbench.sh
+          ./.github/workflows/install_gtest_parallel.sh
           if [[ $OS == ubuntu* ]]; then
             sudo ./dependencies.sh
             ./.github/workflows/install_grpc.sh
-            ./.github/workflows/install_gtest_parallel.sh
+            ./.github/workflows/install_gbench.sh
           else
             ./dependencies.sh
           fi
@@ -85,9 +77,13 @@ jobs:
           export CPLUS_INCLUDE_PATH=$GITHUB_WORKSPACE/grpc/third_party/abseil-cpp
           mkdir build && cd build
           if [[ $OS == ubuntu-latest ]]; then
-            cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            if [[ $SANITIZER == 'ASAN' ]]; then
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_SHARDING=On ..
+            else
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            fi
           elif [[ $OS == macos* ]]; then
-            cmake ..
+            cmake -DGH_CI_OSX=ON ..
           else
             cmake -DENABLE_GRPC=ON ..
           fi
@@ -119,8 +115,6 @@ jobs:
         include:
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
             test: 'C++'
           - os: ubuntu-latest
             sanitizer: ASAN
@@ -141,6 +135,7 @@ jobs:
       CC: ${{matrix.cc}}
       CXX: ${{matrix.cxx}}
       TEST: ${{matrix.test}}
+      SANITIZER: ${{matrix.sanitizer}}
     steps:
       - name: Checkout repository
         if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
@@ -178,71 +173,78 @@ jobs:
         if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
         run: |
           if [[ $TEST == 'GO' ]]; then
-            if [[ -z "${{matrix.sanitizer}}" ]]; then
-              go test -timeout 15m ./test/... -bench . -benchmem -benchtime 100ms -seedcount=50000
+            if [[ $SANITIZER == 'ASAN' ]]; then
+              export REINDEXER_GH_CI_ASAN=true
+            fi
+            if [[ -z "$SANITIZER" ]]; then
+              go test -timeout 15m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
             else
-              go test -timeout 35m ./test/... -bench . -benchmem -benchtime 100ms -seedcount=50000
+              export TSAN_OPTIONS="halt_on_error=1"
+              export RX_IVF_OMP_THREADS=1
+              export REINDEXER_GH_CI_TSAN=1
+              go test -timeout 35m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000 -tags tiny_vectors
             fi
           else
             cd build
             ctest --verbose
           fi
 
-  test-pyreindexer:
-    strategy:
-      matrix:
-        os: [ubuntu-22.04, ubuntu-24.04]
-      fail-fast: false
-    runs-on: ${{matrix.os}}
-    needs: build
-    if: always()
-    env:
-      OS: ${{matrix.os}}
-    steps:
-      - name: Download ${{matrix.os}} Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{matrix.os}}
-      - name: 'Untar Artifacts'
-        run: tar -xvf artifacts.tar
-      - name: Prepare Environment
-        run: |
-          if [[ $OS == ubuntu* ]]; then
-            sudo ./dependencies.sh
-            python3 -m pip install setuptools build
-          else
-            ./dependencies.sh
-          fi
-      - name: Install Reindexer
-        run: |
-          cd build
-          if [[ $OS == ubuntu* ]]; then
-            sudo dpkg -i reindexer-dev*.deb
-            sudo apt-get install -f
-            sudo dpkg -i reindexer-server*.deb
-            sudo apt-get install -f
-          else
-            for f in reindexer-*.tar.gz; do tar -xvzf "$f"; done
-            cp -R ./usr/local/include/reindexer /usr/local/include/reindexer
-            cp -R ./usr/local/lib/reindexer /usr/local/lib/reindexer
-            cp ./usr/local/lib/libreindexer.a /usr/local/lib/libreindexer.a
-            cp ./usr/local/lib/libreindexer_server_library.a /usr/local/lib/libreindexer_server_library.a
-            cp ./usr/local/lib/libreindexer_server_resources.a /usr/local/lib/libreindexer_server_resources.a
-            cp ./usr/local/lib/pkgconfig/libreindexer.pc /usr/local/lib/pkgconfig/libreindexer.pc
-            cp ./usr/local/lib/pkgconfig/libreindexer_server.pc /usr/local/lib/pkgconfig/libreindexer_server.pc
-            cp ./usr/local/bin/reindexer_server /usr/local/bin/reindexer_server
-            cp ./usr/local/etc/reindexer.conf.pkg /usr/local/etc/reindexer.conf.pkg
-          fi
-      - name: Clone PyReindexer
-        uses: actions/checkout@v4
-        with:
-          repository: restream/reindexer-py
-      - name: Install PyReindexer
-        run: |
-          python -m build
-          python -m pip install .
-      - name: Test PyReindexer
-        run: |
-          cd pyreindexer
-          ../.github/workflows/test.sh
+# TODO: Reenable after binding's compatibility fix
+#  test-pyreindexer:
+#    strategy:
+#      matrix:
+#        os: [ubuntu-22.04, ubuntu-24.04]
+#      fail-fast: false
+#    runs-on: ${{matrix.os}}
+#    needs: build
+#    if: always()
+#    env:
+#      OS: ${{matrix.os}}
+#    steps:
+#      - name: Download ${{matrix.os}} Artifacts
+#        uses: actions/download-artifact@v4
+#        with:
+#          name: ${{matrix.os}}
+#      - name: 'Untar Artifacts'
+#        run: tar -xvf artifacts.tar
+#      - name: Prepare Environment
+#        run: |
+#          if [[ $OS == ubuntu* ]]; then
+#            sudo ./dependencies.sh
+#            python3 -m pip install setuptools build
+#          else
+#            ./dependencies.sh
+#          fi
+#      - name: Install Reindexer
+#        run: |
+#          cd build
+#          if [[ $OS == ubuntu* ]]; then
+#            sudo dpkg -i reindexer-4-dev*.deb
+#            sudo apt-get install -f
+#            sudo dpkg -i reindexer-4-server*.deb
+#            sudo apt-get install -f
+#          else
+#            for f in reindexer-*.tar.gz; do tar -xvzf "$f"; done
+#            cp -R ./usr/local/include/reindexer /usr/local/include/reindexer
+#            cp -R ./usr/local/lib/reindexer /usr/local/lib/reindexer
+#            cp ./usr/local/lib/libreindexer.a /usr/local/lib/libreindexer.a
+#            cp ./usr/local/lib/libreindexer_server_library.a /usr/local/lib/libreindexer_server_library.a
+#            cp ./usr/local/lib/libreindexer_server_resources.a /usr/local/lib/libreindexer_server_resources.a
+#            cp ./usr/local/lib/pkgconfig/libreindexer.pc /usr/local/lib/pkgconfig/libreindexer.pc
+#            cp ./usr/local/lib/pkgconfig/libreindexer_server.pc /usr/local/lib/pkgconfig/libreindexer_server.pc
+#            cp ./usr/local/bin/reindexer_server /usr/local/bin/reindexer_server
+#            cp ./usr/local/etc/reindexer.conf.pkg /usr/local/etc/reindexer.conf.pkg
+#          fi
+#      - name: Clone PyReindexer
+#        uses: actions/checkout@v4
+#        with:
+#          repository: restream/reindexer-py
+#      - name: Install PyReindexer
+#        run: |
+#          python -m build
+#          python -m pip install .
+#      - name: Test PyReindexer
+#        run: |
+#          cd pyreindexer
+#          ../.github/workflows/test.sh
 
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,8 @@ __pycache__
 *.so
 .pytest_cache/
 qa_tests/logs/
+qa_tests/logs/
+qa_tests/venv/
 
 #ignore directory with compile commands
-.cache
+.cache
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,10 +1,11 @@
-cmake_minimum_required(VERSION 3.10..3.13)
+cmake_minimum_required(VERSION 3.18)
 
 project(reindexer)
 
 enable_testing()
 set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_PCH_WARN_INVALID OFF)
 set(REINDEXER_SOURCE_PATH ${PROJECT_SOURCE_DIR}/cpp_src)
 
 add_subdirectory(cpp_src)
diff --git a/appveyor.yml b/appveyor.yml