Add patch to support multi vector in faiss

Signed-off-by: Heemin Kim <[email protected]>

.github/workflows/CI.yml

@@ -38,6 +38,14 @@ jobs:
         with:
           submodules: true
 
+      # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here.
+      - name: Apply Git Patch
+        run: |
+          cd jni/external/faiss
+          git config --global core.autocrlf input
+          git apply --ignore-space-change --ignore-whitespace --3way ${{ github.workspace }}/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
+        working-directory: ${{ github.workspace }}
+
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:

CHANGELOG.md

@@ -28,3 +28,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319)
 * Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319)
 ### Refactoring
+* Add patch to support multi vector in faiss [#1358](https://github.com/opensearch-project/k-NN/pull/1358)

jni/CMakeLists.txt

@@ -58,6 +58,13 @@ if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL aarch64)
 elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64)
     set(MACH_ARCH x64)
 endif()
+
+# Set git config to make `git apply` works across platforms
+if (NOT "${WIN32}" STREQUAL "")
+    execute_process(COMMAND git config --global core.autocrlf input)
+else()
+    execute_process(COMMAND git config --global core.autocrlf true)
+endif()
 # ----------------------------------------------------------------------------
 
 # ---------------------------------- COMMON ----------------------------------
@@ -79,7 +86,7 @@ list(APPEND TARGET_LIBS ${TARGET_LIB_COMMON})
 # ---------------------------------- NMSLIB ----------------------------------
 if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON)
     # Check if nmslib exists
-    find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib)
+    find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib NO_DEFAULT_PATH)
 
     # If not, pull the updated submodule
     if (NOT EXISTS ${NMS_REPO_DIR})
@@ -134,14 +141,20 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S
     find_package(LAPACK REQUIRED)
 
     # Check if faiss exists
-    find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss)
+    find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss NO_DEFAULT_PATH)
 
-    # If not, pull the updated submodule
+    # If not, pull the updated submodule and apply patches
     if (NOT EXISTS ${FAISS_REPO_DIR})
         message(STATUS "Could not find faiss. Pulling updated submodule.")
         execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
     endif ()
 
+    message(STATUS "Applying custom patches.")
+    execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE)
+    if(RESULT_CODE)
+        message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}")
+    endif()
+
     set(FAISS_ENABLE_GPU OFF)
     set(FAISS_ENABLE_PYTHON OFF)
     add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/faiss EXCLUDE_FROM_ALL)

jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch

@@ -0,0 +1,31 @@
+From 02c42910b123f00270846cb06997a5f44756a15c Mon Sep 17 00:00:00 2001
+From: Heemin Kim <[email protected]>
+Date: Mon, 11 Dec 2023 10:04:16 -0800
+Subject: [PATCH] Update faiss library with an extension point
+
+Signed-off-by: Heemin Kim <[email protected]>
+---
+ .gitmodules        | 2 +-
+ jni/external/faiss | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/.gitmodules b/.gitmodules
+index 631d4f5..a1cfcfa 100644
+--- a/.gitmodules
++++ b/.gitmodules
+@@ -3,4 +3,4 @@
+ 	url = https://github.com/nmslib/nmslib.git
+ [submodule "jni/external/faiss"]
+ 	path = jni/external/faiss
+-	url = https://github.com/facebookresearch/faiss.git
++	url = [email protected]:heemin32/faiss.git 
+diff --git a/jni/external/faiss b/jni/external/faiss
+index 3219e3d..3081f4c 160000
+--- a/jni/external/faiss
++++ b/jni/external/faiss
+@@ -1 +1 @@
+-Subproject commit 3219e3d12e6fc36dfdfe17d4cf238ef70bf89568
++Subproject commit 3081f4c7f088c156914beb8acc5bc513e7278a64
+-- 
+2.39.3 (Apple Git-145)
+