feat: enable toggling index integrity check on index creation and upd…

…ate vec-447 (#67) * feat: enable toggling index integrity check on index creation and update vec-447 * ci: use avs 1.0.0 for integration tests
aerospike · Dec 11, 2024 · 5db9219 · 5db9219
1 parent 6a4ebd2
commit 5db9219
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 14 deletions.
diff --git a/.github/workflows/extensive_vector_search_tests.yml b/.github/workflows/extensive_vector_search_tests.yml
@@ -64,7 +64,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5

diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml
@@ -62,7 +62,7 @@ jobs:
 
     - name: Run unit tests
       run: |
-        docker run -d  --network=host -p 5000:5000 --name aerospike-vector-search -v ./aerospike-vector-search.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml -v ./features.conf:/etc/aerospike-vector-search/features.conf aerospike/aerospike-vector-search:0.11.1
+        docker run -d  --network=host -p 5000:5000 --name aerospike-vector-search -v ./aerospike-vector-search.yml:/etc/aerospike-vector-search/aerospike-vector-search.yml -v ./features.conf:/etc/aerospike-vector-search/features.conf aerospike/aerospike-vector-search:1.0.0
     
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
@@ -146,7 +146,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000   -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000   -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5
@@ -228,7 +228,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000   -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000   -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5
@@ -309,7 +309,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000   -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000   -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5
@@ -393,7 +393,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5
@@ -471,7 +471,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5
@@ -550,7 +550,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5
@@ -624,7 +624,7 @@ jobs:
     - name: Run unit tests
       run: |
 
-        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:0.11.1
+        docker run -d --name aerospike-vector-search --network=host -p 5000:5000  -v $(pwd):/etc/aerospike-vector-search aerospike/aerospike-vector-search:1.0.0
         docker run -d --name aerospike -p 3000:3000 -v .:/etc/aerospike aerospike/aerospike-server-enterprise:latest 
 
         sleep 5

diff --git a/src/aerospike_vector_search/shared/conversions.py b/src/aerospike_vector_search/shared/conversions.py
@@ -105,6 +105,7 @@ def fromIndexDefintion(input_data: types_pb2.IndexDefinition) -> types.IndexDefi
             m=input_data.hnswParams.m,
             ef_construction=input_data.hnswParams.efConstruction,
             ef=input_data.hnswParams.ef,
+            enable_vector_integrity_check=input_data.hnswParams.enableVectorIntegrityCheck,
             batching_params=types.HnswBatchingParams(
                 max_index_records=input_data.hnswParams.batchingParams.maxIndexRecords,
                 index_interval=input_data.hnswParams.batchingParams.indexInterval,

diff --git a/src/aerospike_vector_search/types.py b/src/aerospike_vector_search/types.py
@@ -539,6 +539,10 @@ class HnswParams(object):
 
     :param batching_params: Parameters related to configuring batch processing, such as the maximum number of records per batch and batching interval. Defaults to HnswBatchingParams().
     :type batching_params: Optional[HnswBatchingParams]
+
+    :param enable_vector_integrity_check: Verifies if the underlying vector has changed before returning the kANN result.
+    :type enable_vector_integrity_check: Optional[bool]
+
     """
 
     def __init__(
@@ -552,7 +556,7 @@ def __init__(
         index_caching_params: HnswCachingParams = HnswCachingParams(),
         healer_params: HnswHealerParams = HnswHealerParams(),
         merge_params: HnswIndexMergeParams = HnswIndexMergeParams(),
-        enable_vector_integrity_check : bool = True,
+        enable_vector_integrity_check : Optional[bool] = None,
         record_caching_params : HnswCachingParams = HnswCachingParams()
     ) -> None:
         self.m = m
@@ -580,7 +584,7 @@ def _to_pb2(self):
         if self.max_mem_queue_size:
             params.maxMemQueueSize = self.max_mem_queue_size
 
-        if self.enable_vector_integrity_check:
+        if self.enable_vector_integrity_check is not None:
             params.enableVectorIntegrityCheck = self.enable_vector_integrity_check
 
         params.batchingParams.CopyFrom(self.batching_params._to_pb2())

diff --git a/tests/standard/aio/test_admin_client_index_create.py b/tests/standard/aio/test_admin_client_index_create.py
@@ -335,7 +335,7 @@ async def test_index_create_with_sets(session_admin_client, test_case, random_na
                 m=32,
                 ef_construction=200,
                 ef=400,
-                enable_vector_integrity_check= True,
+                enable_vector_integrity_check= False,
             ),
             index_labels=None,
             index_storage=None,
@@ -365,6 +365,7 @@ async def test_index_create_with_sets(session_admin_client, test_case, random_na
             sets=None,
             index_params=types.HnswParams(
                 m=8,
+                enable_vector_integrity_check= True,
             ),
             index_labels=None,
             index_storage=None,
@@ -377,7 +378,8 @@ async def test_index_create_with_sets(session_admin_client, test_case, random_na
             vector_distance_metric=None,
             sets=None,
             index_params=types.HnswParams(
-                batching_params=types.HnswBatchingParams(max_index_records=2000, index_interval=20000, max_reindex_records=1500, reindex_interval=70000)
+                batching_params=types.HnswBatchingParams(max_index_records=2000, index_interval=20000, max_reindex_records=1500, reindex_interval=70000),
+                enable_vector_integrity_check= True
             ),
             index_labels=None,
             index_storage=None,
@@ -402,6 +404,7 @@ async def test_index_create_with_sets(session_admin_client, test_case, random_na
                     index_parallelism=10,
                     reindex_parallelism=3
                 ),
+                enable_vector_integrity_check= True
             ),
             index_labels=None,
             index_storage=None,

diff --git a/tests/standard/aio/test_admin_client_index_update.py b/tests/standard/aio/test_admin_client_index_update.py
@@ -57,6 +57,7 @@ def __init__(
                 index_caching_params=types.HnswCachingParams(max_entries=10, expiry=3000),
                 merge_params=types.HnswIndexMergeParams(index_parallelism=10, reindex_parallelism=3),
                 healer_params=types.HnswHealerParams(max_scan_rate_per_node=80),
+                enable_vector_integrity_check=False,
             ),
             timeout=None,
         ),
@@ -129,5 +130,7 @@ async def test_index_update_async(session_admin_client, test_case):
         assert result["hnsw_params"]["healer_params"][
                    "max_scan_rate_per_node"] == test_case.hnsw_index_update.healer_params.max_scan_rate_per_node
 
+    assert result["hnsw_params"]["enable_vector_integrity_check"] == test_case.hnsw_index_update.enable_vector_integrity_check
+
     # Clean up by dropping the index after the test
     await drop_specified_index(session_admin_client, test_case.namespace, trimmed_random)
diff --git a/tests/standard/sync/test_admin_client_index_create.py b/tests/standard/sync/test_admin_client_index_create.py
@@ -357,7 +357,7 @@ def test_index_create_with_sets(session_admin_client, test_case, random_name):
                 m=32,
                 ef_construction=200,
                 ef=400,
-                enable_vector_integrity_check = True,
+                enable_vector_integrity_check = False,
             ),
             index_labels=None,
             index_storage=None,
@@ -397,6 +397,7 @@ def test_index_create_with_sets(session_admin_client, test_case, random_name):
             sets="demo",
             index_params=types.HnswParams(
                 batching_params=types.HnswBatchingParams(max_index_records=2000, index_interval=20000, max_reindex_records=1500, reindex_interval=70000),
+                enable_vector_integrity_check= True
             ),
             index_labels=None,
             index_storage=None,
@@ -421,6 +422,7 @@ def test_index_create_with_sets(session_admin_client, test_case, random_name):
                     index_parallelism=10,
                     reindex_parallelism=3
                 ),
+                enable_vector_integrity_check= True
             ),
             index_labels=None,
             index_storage=None,

diff --git a/tests/standard/sync/test_admin_client_index_update.py b/tests/standard/sync/test_admin_client_index_update.py
@@ -42,6 +42,7 @@ def __init__(
                 index_caching_params=types.HnswCachingParams(max_entries=10, expiry=3000),
                 merge_params=types.HnswIndexMergeParams(index_parallelism=10,reindex_parallelism=3),
                 healer_params=types.HnswHealerParams(max_scan_rate_per_node=80),
+                enable_vector_integrity_check=False,
             ),
             timeout=None,
         ),
@@ -102,5 +103,7 @@ def test_index_update(session_admin_client, test_case):
     if test_case.hnsw_index_update.healer_params:
         assert result["hnsw_params"]["healer_params"]["max_scan_rate_per_node"] == test_case.hnsw_index_update.healer_params.max_scan_rate_per_node
 
+    assert result["hnsw_params"]["enable_vector_integrity_check"] == test_case.hnsw_index_update.enable_vector_integrity_check
+
     # Clean up by dropping the index after the test
     drop_specified_index(session_admin_client, test_case.namespace, trimmed_random)