Update faiss commit and resolve conflict (#1443)

Signed-off-by: Heemin Kim <[email protected]> (cherry picked from commit ced1ff2)
opensearch-project · Feb 13, 2024 · 426a879 · 426a879
1 parent 5e8a1c6
commit 426a879
Show file tree

Hide file tree

Showing 20 changed files with 1,041 additions and 1,083 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,4 +19,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Infrastructure
 ### Documentation
 ### Maintenance
+* Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443)
 ### Refactoring
diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt
@@ -181,17 +181,15 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S
         ${TARGET_LIB_FAISS} SHARED
         ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/utils/BitSet.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollector.cpp
-        ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp)
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_util.cpp
+    )
     target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX)
     target_include_directories(${TARGET_LIB_FAISS} PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/include
-        ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss
-        ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss/utils
         $ENV{JAVA_HOME}/include
         $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}
-        ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss)
+        ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss
+    )
     set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT})
     set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
@@ -228,12 +226,9 @@ if ("${WIN32}" STREQUAL "")
         add_executable(
                 jni_test
                 tests/faiss_wrapper_test.cpp
+                tests/faiss_util_test.cpp
                 tests/nmslib_wrapper_test.cpp
                 tests/test_util.cpp
-                tests/knn_extension/faiss/utils/BitSetTest.cpp
-                tests/knn_extension/faiss/utils/HeapTest.cpp
-                tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp
-                tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp
                 )
 
         target_link_libraries(

diff --git a/jni/external/faiss b/jni/external/faiss
diff --git a/jni/include/faiss_util.h b/jni/include/faiss_util.h
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The OpenSearch Contributors require contributions made to
+// this file be licensed under the Apache-2.0 license or a
+// compatible open source license.
+//
+// Modifications Copyright OpenSearch Contributors. See
+// GitHub history for details.
+
+/**
+ * This file contains util methods which are free of JNI to be used in faiss_wrapper.cpp
+ */
+
+#ifndef OPENSEARCH_KNN_FAISS_UTIL_H
+#define OPENSEARCH_KNN_FAISS_UTIL_H
+
+#include "faiss/impl/IDGrouper.h"
+#include <memory>
+
+namespace faiss_util {
+    std::unique_ptr<faiss::IDGrouperBitmap> buildIDGrouperBitmap(int *parentIdsArray,  int parentIdsLength, std::vector<uint64_t>* bitmap);
+};
+
+
+#endif //OPENSEARCH_KNN_FAISS_UTIL_H
diff --git a/jni/include/knn_extension/faiss/MultiVectorResultCollector.h b/jni/include/knn_extension/faiss/MultiVectorResultCollector.h
diff --git a/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h b/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h
diff --git a/jni/include/knn_extension/faiss/utils/BitSet.h b/jni/include/knn_extension/faiss/utils/BitSet.h
+1 −0		CHANGELOG.md
+2 −0		benchs/link_and_code/README.md
+1 −0		contrib/evaluation.py
+17 −10		faiss/IndexAdditiveQuantizer.cpp
+2 −2		faiss/IndexAdditiveQuantizerFastScan.cpp
+13 −10		faiss/IndexBinaryHNSW.cpp
+81 −150		faiss/IndexFastScan.cpp
+9 −8		faiss/IndexFastScan.h
+64 −284		faiss/IndexHNSW.cpp
+7 −45		faiss/IndexHNSW.h
+7 −11		faiss/IndexIVFAdditiveQuantizerFastScan.cpp
+1 −2		faiss/IndexIVFAdditiveQuantizerFastScan.h
+412 −466		faiss/IndexIVFFastScan.cpp
+44 −37		faiss/IndexIVFFastScan.h
+1 −1		faiss/IndexIVFPQ.cpp
+8 −8		faiss/IndexIVFPQFastScan.cpp
+1 −2		faiss/IndexIVFPQFastScan.h
+0 −1		faiss/impl/AuxIndexStructures.h
+41 −45		faiss/impl/HNSW.cpp
+7 −8		faiss/impl/HNSW.h
+1 −1		faiss/impl/Quantizer.h
+224 −168		faiss/impl/ResultHandler.h
+3 −0		faiss/impl/pq4_fast_scan.cpp
+7 −6		faiss/impl/pq4_fast_scan.h
+51 −37		faiss/impl/pq4_fast_scan_search_1.cpp
+37 −46		faiss/impl/pq4_fast_scan_search_qbs.cpp
+374 −202		faiss/impl/simd_result_handlers.h
+8 −0		faiss/python/swigfaiss.swig
+59 −48		faiss/utils/distances.cpp
+4 −4		faiss/utils/distances_fused/avx512.cpp
+1 −1		faiss/utils/distances_fused/avx512.h
+1 −1		faiss/utils/distances_fused/distances_fused.cpp
+1 −1		faiss/utils/distances_fused/distances_fused.h
+4 −4		faiss/utils/distances_fused/simdlib_based.cpp
+1 −1		faiss/utils/distances_fused/simdlib_based.h
+74 −5		tests/test_fast_scan_ivf.py
+426 −0		tests/test_graph_based.py
+0 −400		tests/test_index.py