[MOD-6738] IndexComputer

.github/workflows/flow-temp.yml

@@ -9,19 +9,19 @@ name: temporary testing
 
 on:
   push:
-    branches-ignore: ['**'] # ignore all branches. Comment this line to run your workflow below on every push.
+    # branches-ignore: ['**'] # ignore all branches. Comment this line to run your workflow below on every push.
 jobs:
   jammy:
     uses: ./.github/workflows/task-unit-test.yml
     with:
       container: ubuntu:jammy
       run-valgrind: true
-  alpine3:
-    uses: ./.github/workflows/task-unit-test.yml
-    with:
-      container: alpine:3
-      pre-checkout-script: apk add bash
-      run-valgrind: true
+  # alpine3:
+  #   uses: ./.github/workflows/task-unit-test.yml
+  #   with:
+  #     container: alpine:3
+  #     pre-checkout-script: apk add bash
+  #     run-valgrind: true
   # focal:
   #   uses: ./.github/workflows/task-unit-test.yml
   #   with:

src/VecSim/algorithms/brute_force/bfm_batch_iterator.h

@@ -27,13 +27,12 @@ class BFM_BatchIterator : public BF_BatchIterator<DataType, DistType> {
 
         idType curr_id = 0;
         auto vectors_it = this->index->getVectorsIterator();
-        auto DistFunc = this->index->getDistFunc();
         while (auto *vector = vectors_it->next()) {
             // Compute the scores for every vector and extend the scores array.
             if (VECSIM_TIMEOUT(this->getTimeoutCtx())) {
                 return VecSim_QueryReply_TimedOut;
             }
-            auto score = DistFunc(vector, this->getQueryBlob(), this->index->getDim());
+            auto score = this->index->calcDistance(vector, this->getQueryBlob());
             labelType curr_label = this->index->getVectorLabel(curr_id);
             auto curr_pair = tmp_scores.find(curr_label);
             // For each score, emplace or update the score of the label.

src/VecSim/algorithms/brute_force/bfs_batch_iterator.h

@@ -25,13 +25,12 @@ class BFS_BatchIterator : public BF_BatchIterator<DataType, DistType> {
 
         idType curr_id = 0;
         auto vectors_it = this->index->getVectorsIterator();
-        auto DistFunc = this->index->getDistFunc();
         while (auto *vector = vectors_it->next()) {
             // Compute the scores for every vector and extend the scores array.
             if (VECSIM_TIMEOUT(this->getTimeoutCtx())) {
                 return VecSim_QueryReply_TimedOut;
             }
-            auto score = DistFunc(vector, this->getQueryBlob(), this->index->getDim());
+            auto score = this->index->calcDistance(vector, this->getQueryBlob());
             this->scores.emplace_back(score, this->index->getVectorLabel(curr_id));
             ++curr_id;
         }

src/VecSim/algorithms/brute_force/brute_force.h

@@ -35,7 +35,8 @@ class BruteForceIndex : public VecSimIndexAbstract<DataType, DistType> {
     idType count;
 
 public:
-    BruteForceIndex(const BFParams *params, const AbstractIndexInitParams &abstractInitParams);
+    BruteForceIndex(const BFParams *params, const AbstractIndexInitParams &abstractInitParams,
+                    const IndexComponents<DataType, DistType> &components);
 
     size_t indexSize() const override;
     size_t indexCapacity() const override;
@@ -138,8 +139,9 @@ class BruteForceIndex : public VecSimIndexAbstract<DataType, DistType> {
 /******************** Ctor / Dtor **************/
 template <typename DataType, typename DistType>
 BruteForceIndex<DataType, DistType>::BruteForceIndex(
-    const BFParams *params, const AbstractIndexInitParams &abstractInitParams)
-    : VecSimIndexAbstract<DataType, DistType>(abstractInitParams),
+    const BFParams *params, const AbstractIndexInitParams &abstractInitParams,
+    const IndexComponents<DataType, DistType> &components)
+    : VecSimIndexAbstract<DataType, DistType>(abstractInitParams, components),
       idToLabelMapping(this->allocator), count(0) {
     assert(VecSimType_sizeof(this->vecType) == sizeof(DataType));
     // Round up the initial capacity to the nearest multiple of the block size.
@@ -153,6 +155,7 @@ BruteForceIndex<DataType, DistType>::BruteForceIndex(
 
 template <typename DataType, typename DistType>
 void BruteForceIndex<DataType, DistType>::appendVector(const void *vector_data, labelType label) {
+    auto processed_blob = this->preprocessForStorage(vector_data);
     // Give the vector new id and increase count.
     idType id = this->count++;
 
@@ -161,7 +164,7 @@ void BruteForceIndex<DataType, DistType>::appendVector(const void *vector_data,
         growByBlock();
     }
     // add vector data to vector raw data container
-    vectors->addElement(vector_data, id);
+    vectors->addElement(processed_blob.get(), id);
 
     // add label to idToLabelMapping
     setVectorLabel(id, label);
@@ -230,6 +233,8 @@ BruteForceIndex<DataType, DistType>::topKQuery(const void *queryBlob, size_t k,
         return rep;
     }
 
+    auto processed_query_ptr = this->preprocessQuery(queryBlob);
+    const void *processed_query = processed_query_ptr.get();
     DistType upperBound = std::numeric_limits<DistType>::lowest();
     vecsim_stl::abstract_priority_queue<DistType, labelType> *TopCandidates =
         getNewMaxPriorityQueue();
@@ -243,7 +248,7 @@ BruteForceIndex<DataType, DistType>::topKQuery(const void *queryBlob, size_t k,
             delete TopCandidates;
             return rep;
         }
-        auto score = this->distFunc(vector, queryBlob, this->dim);
+        auto score = this->calcDistance(vector, processed_query);
         // If we have less than k or a better score, insert it.
         if (score < upperBound || TopCandidates->size() < k) {
             TopCandidates->emplace(score, getVectorLabel(curr_id));
@@ -270,6 +275,7 @@ template <typename DataType, typename DistType>
 VecSimQueryReply *
 BruteForceIndex<DataType, DistType>::rangeQuery(const void *queryBlob, double radius,
                                                 VecSimQueryParams *queryParams) const {
+    auto processed_query_ptr = this->preprocessQuery(queryBlob);
     auto rep = new VecSimQueryReply(this->allocator);
     void *timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr;
     this->lastMode = RANGE_QUERY;
@@ -281,12 +287,13 @@ BruteForceIndex<DataType, DistType>::rangeQuery(const void *queryBlob, double ra
     DistType radius_ = DistType(radius);
     auto vectors_it = vectors->getIterator();
     idType curr_id = 0;
+    const void *processed_query = processed_query_ptr.get();
     while (vectors_it->hasNext()) {
         if (VECSIM_TIMEOUT(timeoutCtx)) {
             rep->code = VecSim_QueryReply_TimedOut;
             break;
         }
-        auto score = this->distFunc(vectors_it->next(), queryBlob, this->dim);
+        auto score = this->calcDistance(vectors_it->next(), processed_query);
         if (score <= radius_) {
             res_container->emplace(getVectorLabel(curr_id), score);
         }
@@ -342,8 +349,10 @@ template <typename DataType, typename DistType>
 VecSimBatchIterator *
 BruteForceIndex<DataType, DistType>::newBatchIterator(const void *queryBlob,
                                                       VecSimQueryParams *queryParams) const {
-    auto *queryBlobCopy = this->allocator->allocate(sizeof(DataType) * this->dim);
+    auto *queryBlobCopy =
+        this->allocator->allocate_aligned(this->dataSize, this->preprocessors->getAlignment());
     memcpy(queryBlobCopy, queryBlob, this->dim * sizeof(DataType));
+    this->preprocessQueryInPlace(queryBlobCopy);
     // Ownership of queryBlobCopy moves to BF_BatchIterator that will free it at the end.
     return newBatchIterator_Instance(queryBlobCopy, queryParams);
 }

src/VecSim/algorithms/brute_force/brute_force_multi.h

@@ -17,13 +17,14 @@ class BruteForceIndex_Multi : public BruteForceIndex<DataType, DistType> {
     vecsim_stl::unordered_map<labelType, vecsim_stl::vector<idType>> labelToIdsLookup;
 
 public:
-    BruteForceIndex_Multi(const BFParams *params, const AbstractIndexInitParams &abstractInitParams)
-        : BruteForceIndex<DataType, DistType>(params, abstractInitParams),
+    BruteForceIndex_Multi(const BFParams *params, const AbstractIndexInitParams &abstractInitParams,
+                          const IndexComponents<DataType, DistType> &components)
+        : BruteForceIndex<DataType, DistType>(params, abstractInitParams, components),
           labelToIdsLookup(this->allocator) {}
 
     ~BruteForceIndex_Multi() = default;
 
-    int addVector(const void *vector_data, labelType label, void *auxiliaryCtx = nullptr) override;
+    int addVector(const void *vector_data, labelType label) override;
     int deleteVector(labelType labelType) override;
     int deleteVectorById(labelType label, idType id) override;
     double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override;
@@ -93,8 +94,7 @@ class BruteForceIndex_Multi : public BruteForceIndex<DataType, DistType> {
 /******************************* Implementation **********************************/
 
 template <typename DataType, typename DistType>
-int BruteForceIndex_Multi<DataType, DistType>::addVector(const void *vector_data, labelType label,
-                                                         void *auxiliaryCtx) {
+int BruteForceIndex_Multi<DataType, DistType>::addVector(const void *vector_data, labelType label) {
     this->appendVector(vector_data, label);
     return 1;
 }
@@ -203,7 +203,7 @@ BruteForceIndex_Multi<DataType, DistType>::getDistanceFrom_Unsafe(labelType labe
 
     DistType dist = std::numeric_limits<DistType>::infinity();
     for (auto id : IDs->second) {
-        DistType d = this->distFunc(this->getDataByInternalId(id), vector_data, this->dim);
+        DistType d = this->calcDistance(this->getDataByInternalId(id), vector_data);
         dist = (dist < d) ? dist : d;
     }
 

src/VecSim/algorithms/brute_force/brute_force_single.h

@@ -18,10 +18,11 @@ class BruteForceIndex_Single : public BruteForceIndex<DataType, DistType> {
 
 public:
     BruteForceIndex_Single(const BFParams *params,
-                           const AbstractIndexInitParams &abstractInitParams);
+                           const AbstractIndexInitParams &abstractInitParams,
+                           const IndexComponents<DataType, DistType> &components);
     ~BruteForceIndex_Single() = default;
 
-    int addVector(const void *vector_data, labelType label, void *auxiliaryCtx = nullptr) override;
+    int addVector(const void *vector_data, labelType label) override;
     int deleteVector(labelType label) override;
     int deleteVectorById(labelType label, idType id) override;
     double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override;
@@ -97,13 +98,14 @@ class BruteForceIndex_Single : public BruteForceIndex<DataType, DistType> {
 
 template <typename DataType, typename DistType>
 BruteForceIndex_Single<DataType, DistType>::BruteForceIndex_Single(
-    const BFParams *params, const AbstractIndexInitParams &abstractInitParams)
-    : BruteForceIndex<DataType, DistType>(params, abstractInitParams),
+    const BFParams *params, const AbstractIndexInitParams &abstractInitParams,
+    const IndexComponents<DataType, DistType> &components)
+    : BruteForceIndex<DataType, DistType>(params, abstractInitParams, components),
       labelToIdLookup(this->allocator) {}
 
 template <typename DataType, typename DistType>
-int BruteForceIndex_Single<DataType, DistType>::addVector(const void *vector_data, labelType label,
-                                                          void *auxiliaryCtx) {
+int BruteForceIndex_Single<DataType, DistType>::addVector(const void *vector_data,
+                                                          labelType label) {
 
     auto optionalID = this->labelToIdLookup.find(label);
     // Check if label already exists, so it is an update operation.
@@ -178,5 +180,5 @@ BruteForceIndex_Single<DataType, DistType>::getDistanceFrom_Unsafe(labelType lab
     }
     idType id = optionalId->second;
 
-    return this->distFunc(this->getDataByInternalId(id), vector_data, this->dim);
+    return this->calcDistance(this->getDataByInternalId(id), vector_data);
 }

src/VecSim/algorithms/hnsw/graph_data.h

@@ -25,7 +25,7 @@ struct ElementLevelData {
     idType links[];
 
     explicit ElementLevelData(std::shared_ptr<VecSimAllocator> allocator)
-        : incomingUnidirectionalEdges(new(allocator) vecsim_stl::vector<idType>(allocator)),
+        : incomingUnidirectionalEdges(new (allocator) vecsim_stl::vector<idType>(allocator)),
           numLinks(0) {}
 
     linkListSize getNumLinks() const { return this->numLinks; }