fix illegal instruction on platform has avx only

.circleci/config.yml

@@ -27,12 +27,12 @@ jobs:
       - restore_cache:
           keys:
             - fork-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }}
-      - run: make asan
+      - run: make debug
       - save_cache:
           key: fork-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }}
           paths:
             - ./build
-      - run: make test_asan_parallel
+      - run: make test_parallel
 
   main-branch-check:
     docker:
@@ -44,9 +44,9 @@ jobs:
       - restore_cache:
           keys:
             - main-ccache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }}
-      - run: make asan
+      - run: make debug
       - save_cache:
           key: main-ccache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }}
           paths:
             - ./build
-      - run: make test_asan_parallel
+      - run: make test_parallel

.github/CODEOWNERS

@@ -10,8 +10,8 @@
 *.md                     @wxyucs @jiaweizone
 
 # GitHub routines
-/.circleci/              @wxyucs
-/.github/                @wxyucs
+/.circleci/              @wxyucs @LHT129
+/.github/                @wxyucs @LHT129
 /docker/                 @wxyucs
 /.*                      @wxyucs
 /LICENSE                 @wxyucs
@@ -22,5 +22,29 @@
 # Core codes, maybe assign reviewers later
 /src/                    @jiaweizone
 
+# index codes
+/src/index               @inabao @wxyucs @LHT129 @ShawnShawnYou
+
+# datacell codes
+/src/data_cell/          @LHT129 @ShawnShawnYou
+
+# io codes
+/src/io/                 @LHT129
+
+# quantization codes
+/src/quantization/       @LHT129 @ShawnShawnYou
+
+# impl codes (conjugate graph)
+/src/impl/               @ShawnShawnYou
+
+# hnswlib codes
+/src/algorithm/          @inabao @wxyucs
+
 # All the SIMD related codes
 /src/simd/               @LHT129 @ShawnShawnYou
+
+# pybinds
+/python_bindings/        @wxyucs @LHT129
+
+# scripts
+/scripts/                @LHT129 @wxyucs @inabao

.github/workflows/DCO.yml

@@ -0,0 +1,15 @@
+name: DCO Check
+
+on: 
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  dco-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Validate DCO
+        uses: tisonkun/[email protected]

.github/workflows/build_test.yml

@@ -0,0 +1,39 @@
+name: build & test
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  clang-format-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Clang format
+        run: sudo apt-get install clang-format
+      - name: Run Clang format check
+        run: make fmt
+
+  build:
+    runs-on: ubuntu-latest
+    container:
+      image: vsaglib/vsag:ubuntu
+    steps:
+      - uses: actions/checkout@v4
+      - name: Load Cache
+        uses: actions/[email protected]
+        with:
+          path: ./build/
+          key: build-${{ hashFiles('./CMakeLists.txt') }}-${{ hashFiles('./.circleci/fresh_ci_cache.commit') }}
+      - name: Make Asan
+        run: make asan
+      - name: Save Cache
+        uses: actions/[email protected]
+        with:
+          path: ./build/
+          key: build-${{ hashFiles('./CMakeLists.txt') }}-${{ hashFiles('./.circleci/fresh_ci_cache.commit') }}
+      - name: Test
+        run: make test_asan

Makefile

@@ -22,7 +22,7 @@ help:                    ## Show the help.
 
 .PHONY: debug
 debug:                   ## Build vsag with debug options.
-	cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_CCACHE=ON
+	cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_CCACHE=ON -DENABLE_ASAN=OFF
 	cmake --build build --parallel ${COMPILE_JOBS}
 
 .PHONY: release
@@ -57,14 +57,19 @@ test:                    ## Build and run unit tests.
 	./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
 	./build/mockimpl/tests_mockimpl -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
 
+.PHONY: test_parallel
+test_parallel: debug
+	@./scripts/test_parallel_bg.sh
+	./build/mockimpl/tests_mockimpl -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
+
 .PHONY: asan
 asan:                    ## Build with AddressSanitizer option.
 	cmake ${VSAG_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON -DENABLE_CCACHE=ON
 	cmake --build build --parallel ${COMPILE_JOBS}
 
 .PHONY: test_asan_parallel
 test_asan_parallel: asan ## Run unit tests parallel with AddressSanitizer option.
-	@./scripts/test_asan_bg.sh
+	@./scripts/test_parallel_bg.sh
 	./build/mockimpl/tests_mockimpl -d yes ${UT_FILTER} --allow-running-no-tests ${UT_SHARD}
 
 .PHONY: test_asan

examples/cpp/simple_hgraph_sq8.cpp

@@ -44,7 +44,9 @@ main(int argc, char** argv) {
         "metric_type": "l2",
         "dim": 128,
         "index_param": {
-            "base_quantization_type": "sq8"
+            "base_quantization_type": "sq8",
+            "max_degree": 26,
+            "ef_construction": 100
         }
     }
     )";

include/vsag/constants.h

@@ -99,5 +99,6 @@ extern const char* const SERIALIZE_VERSION;
 extern const char* const HGRAPH_USE_REORDER;
 extern const char* const HGRAPH_BASE_QUANTIZATION_TYPE;
 extern const char* const HGRAPH_GRAPH_MAX_DEGREE;
+extern const char* const HGRAPH_BUILD_EF_CONSTRUCTION;
 
 }  // namespace vsag

scripts/test_asan_bg.sh → scripts/test_parallel_bg.sh

@@ -8,22 +8,25 @@ othertag=""
 
 mkdir ./log
 
-./build/tests/unittests -d yes ${UT_FILTER} --allow-running-no-tests 2>&1 | tee ./log/unittest.log
-exit_codes+=($?)
+./build/tests/unittests -d yes ${UT_FILTER} --allow-running-no-tests &> "./log/unittest.log" &
+pids+=($!)
+tail -f "./log/unittest.log" &
 logger_files+=("./log/unittest.log")
 
 for tag in ${parallel_tags}
 do
   othertag="~"${tag}${othertag}
-  ./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests ${tag} 2>&1 | tee ./log/${tag}.log &
+  ./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests &> ./log/${tag}.log &
   pids+=($!)
   logname="./log/"${tag}".log"
   logger_files+=($logname)
+  tail -f ./log/${tag}.log &
 done
 
-./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests ${othertag} 2>&1 | tee ./log/other.log &
+./build/tests/functests -d yes ${UT_FILTER} --allow-running-no-tests ${othertag} &> ./log/other.log &
 pids+=($!)
 logger_files+=("./log/other.log")
+tail -f "./log/other.log" &
 
 for pid in "${pids[@]}"
 do

src/constants.cpp

@@ -100,5 +100,6 @@ const char* const SERIALIZE_VERSION = "VERSION";
 const char* const HGRAPH_USE_REORDER = HGRAPH_USE_REORDER_KEY;
 const char* const HGRAPH_BASE_QUANTIZATION_TYPE = "base_quantization_type";
 const char* const HGRAPH_GRAPH_MAX_DEGREE = "max_degree";
+const char* const HGRAPH_BUILD_EF_CONSTRUCTION = "ef_construction";
 
 };  // namespace vsag

src/data_cell/graph_interface.h

@@ -102,7 +102,7 @@ class GraphInterface {
 
     virtual void
     SetMaximumDegree(uint32_t maximum_degree) {
-        this->max_capacity_ = maximum_degree;
+        this->maximum_degree_ = maximum_degree;
     }
 
     virtual void

src/impl/conjugate_graph.cpp

@@ -27,6 +27,9 @@ ConjugateGraph::AddNeighbor(int64_t from_tag_id, int64_t to_tag_id) {
         return false;
     }
     auto& neighbor_set = conjugate_graph_[from_tag_id];
+    if (neighbor_set.size() >= MAXIMUM_DEGREE) {
+        return false;
+    }
     auto insert_result = neighbor_set.insert(to_tag_id);
     if (!insert_result.second) {
         return false;
@@ -54,6 +57,10 @@ ConjugateGraph::get_neighbors(int64_t from_tag_id) const {
 tl::expected<uint32_t, Error>
 ConjugateGraph::EnhanceResult(std::priority_queue<std::pair<float, size_t>>& results,
                               const std::function<float(int64_t)>& distance_of_tag) const {
+    if (this->is_empty()) {
+        return 0;
+    }
+
     int64_t k = results.size();
     int64_t look_at_k = std::min(LOOK_AT_K, k);
     std::priority_queue<std::pair<float, size_t>> old_results(results);
@@ -205,4 +212,9 @@ ConjugateGraph::Deserialize(StreamReader& in_stream) {
     }
 }
 
-}  // namespace vsag
+bool
+ConjugateGraph::is_empty() const {
+    return (this->memory_usage_ == sizeof(this->memory_usage_) + FOOTER_SIZE);
+}
+
+}  // namespace vsag

src/impl/conjugate_graph.h

@@ -28,6 +28,7 @@
 namespace vsag {
 
 static const int64_t LOOK_AT_K = 20;
+static const int64_t MAXIMUM_DEGREE = 128;
 
 class ConjugateGraph {
 public:
@@ -63,6 +64,9 @@ class ConjugateGraph {
     void
     clear();
 
+    bool
+    is_empty() const;
+
 private:
     uint32_t memory_usage_;
 

src/index/hgraph_index.cpp

@@ -17,6 +17,8 @@
 
 #include <fmt/format-inl.h>
 
+#include <memory>
+
 #include "common.h"
 #include "data_cell/sparse_graph_datacell.h"
 #include "hgraph_zparameters.h"
@@ -80,6 +82,16 @@ HGraphIndex::Init() {
     this->pool_ =
         std::make_shared<hnswlib::VisitedListPool>(this->bottom_graph_->MaxCapacity(), allocator_);
 
+    if (this->index_param_.contains(BUILD_PARAMS_KEY)) {
+        auto& build_params = this->index_param_[BUILD_PARAMS_KEY];
+        if (build_params.contains(BUILD_EF_CONSTRUCTION)) {
+            this->ef_construct_ = build_params[BUILD_EF_CONSTRUCTION];
+        }
+        if (build_params.contains(BUILD_THREAD_COUNT)) {
+            this->build_thread_count_ = build_params[BUILD_THREAD_COUNT];
+        }
+    }
+
     if (this->build_thread_count_ > 1) {
         this->build_pool_ = std::make_unique<progschj::ThreadPool>(this->build_thread_count_);
     }
@@ -233,7 +245,7 @@ HGraphIndex::hnsw_add(const DatasetPtr& data) {
     auto* ids = data->GetIds();
     auto* datas = data->GetFloat32Vectors();
     auto cur_count = this->bottom_graph_->TotalCount();
-    vsag::Vector<std::shared_mutex>(total + cur_count, allocator_).swap(this->neighbors_mutex_);
+    this->resize(total + cur_count);
 
     std::mutex add_mutex;
 
@@ -690,4 +702,14 @@ HGraphIndex::add_one_point(const float* data, int level, InnerIdType inner_id) {
     bottom_graph_->IncreaseTotalCount(1);
 }
 
+void
+HGraphIndex::resize(uint64_t new_size) {
+    auto cur_size = this->bottom_graph_->MaxCapacity();
+    if (new_size > cur_size) {
+        vsag::Vector<std::shared_mutex>(new_size, allocator_).swap(this->neighbors_mutex_);
+        pool_ = std::make_shared<hnswlib::VisitedListPool>(new_size, allocator_);
+        this->bottom_graph_->SetMaxCapacity(new_size);
+    }
+}
+
 }  // namespace vsag

src/index/hgraph_index.h

@@ -240,6 +240,9 @@ class HGraphIndex : public Index {
     void
     hnsw_add(const DatasetPtr& data);
 
+    void
+    resize(uint64_t new_size);
+
     GraphInterfacePtr
     generate_one_route_graph();
 

src/index/hgraph_zparameters.cpp

@@ -26,7 +26,8 @@ namespace vsag {
 const std::unordered_map<std::string, std::vector<std::string>> HGraphParameters::EXTERNAL_MAPPING =
     {{HGRAPH_USE_REORDER, {HGRAPH_USE_REORDER_KEY}},
      {HGRAPH_BASE_QUANTIZATION_TYPE, {HGRAPH_BASE_CODES_KEY, QUANTIZATION_TYPE_KEY}},
-     {HGRAPH_GRAPH_MAX_DEGREE, {HGRAPH_GRAPH_KEY, GRAPH_PARAMS_KEY, GRAPH_PARAM_MAX_DEGREE}}};
+     {HGRAPH_GRAPH_MAX_DEGREE, {HGRAPH_GRAPH_KEY, GRAPH_PARAMS_KEY, GRAPH_PARAM_MAX_DEGREE}},
+     {HGRAPH_BUILD_EF_CONSTRUCTION, {BUILD_PARAMS_KEY, BUILD_EF_CONSTRUCTION}}};
 
 HGraphParameters::HGraphParameters(JsonType& hgraph_param, const IndexCommonParam& common_param)
     : common_param_(common_param) {
@@ -95,7 +96,7 @@ const std::string HGraphParameters::DEFAULT_HGRAPH_PARAMS = format_map(
             "type": "nsw",
             "{GRAPH_PARAMS_KEY}": {
                 "{GRAPH_PARAM_MAX_DEGREE}": 64,
-                "{GRAPH_PARAM_INIT_MAX_CAPACITY}": 2000000
+                "{GRAPH_PARAM_INIT_MAX_CAPACITY}": 1000
             }
         },
         "{HGRAPH_BASE_CODES_KEY}": {
@@ -119,8 +120,8 @@ const std::string HGraphParameters::DEFAULT_HGRAPH_PARAMS = format_map(
             "{QUANTIZATION_TYPE_KEY}": "{QUANTIZATION_TYPE_VALUE_SQ8}",
             "{QUANTIZATION_PARAMS_KEY}": {}
         },
-        "build_params": {
-            "ef_construction": 400,
+        "{BUILD_PARAMS_KEY}": {
+            "{BUILD_EF_CONSTRUCTION}": 400,
             "{BUILD_THREAD_COUNT}": 5
         }
     })",