Move filter test configs to separate folders, replace ec2 paths with …

…vars Signed-off-by: Martin Gaievski <[email protected]>
opensearch-project · Dec 14, 2022 · c6ed5df · c6ed5df
1 parent f0669ca
commit c6ed5df
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 10 deletions.
diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json
@@ -0,0 +1,26 @@
+{
+  "settings": {
+    "index": {
+      "knn": true,
+      "number_of_shards": 24,
+      "number_of_replicas": 1
+    }
+  },
+  "mappings": {
+    "properties": {
+      "target_field": {
+          "type": "knn_vector",
+          "dimension": 128,
+      	  "method": {
+        		"name": "hnsw",
+        		"space_type": "l2",
+        		"engine": "lucene",
+        		"parameters": {
+          			"ef_construction": 256,
+          			"m": 16
+        		}
+        	}
+      }
+    }
+  }
+}
diff --git a/...figs/lucene-hnsw/relaxed-filter-spec.json → ...g/relaxed-filter/relaxed-filter-spec.json b/...figs/lucene-hnsw/relaxed-filter-spec.json → ...g/relaxed-filter/relaxed-filter-spec.json
diff --git a/...nfigs/lucene-hnsw/relaxed-filter-test.yml → ...ng/relaxed-filter/relaxed-filter-test.yml b/...nfigs/lucene-hnsw/relaxed-filter-test.yml → ...ng/relaxed-filter/relaxed-filter-test.yml
@@ -8,13 +8,13 @@ steps:
     index_name: target_index
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/index.json
+    index_spec: [INDEX_SPEC_PATH]/index.json
   - name: ingest_multi_field
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5
     attributes_dataset_name: attributes
     attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ]
   - name: refresh_index
@@ -26,8 +26,8 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5
+    neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5
     neighbors_dataset: neighbors_filter_5
-    filter_spec: /home/ec2-user/[PATH]/relaxed-filter-spec.json
+    filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json
diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json
@@ -0,0 +1,26 @@
+{
+  "settings": {
+    "index": {
+      "knn": true,
+      "number_of_shards": 24,
+      "number_of_replicas": 1
+    }
+  },
+  "mappings": {
+    "properties": {
+      "target_field": {
+          "type": "knn_vector",
+          "dimension": 128,
+      	  "method": {
+        		"name": "hnsw",
+        		"space_type": "l2",
+        		"engine": "lucene",
+        		"parameters": {
+          			"ef_construction": 256,
+          			"m": 16
+        		}
+        	}
+      }
+    }
+  }
+}
diff --git a/.../lucene-hnsw/restrictive-filter-spec.json → ...ctive-filter/restrictive-filter-spec.json b/.../lucene-hnsw/restrictive-filter-spec.json → ...ctive-filter/restrictive-filter-spec.json
diff --git a/...s/lucene-hnsw/restrictive-filter-test.yml → ...ictive-filter/restrictive-filter-test.yml b/...s/lucene-hnsw/restrictive-filter-test.yml → ...ictive-filter/restrictive-filter-test.yml
@@ -8,13 +8,13 @@ steps:
     index_name: target_index
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/index.json
+    index_spec: [INDEX_SPEC_PATH]/index.json
   - name: ingest_multi_field
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5
     attributes_dataset_name: attributes
     attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ]
   - name: refresh_index
@@ -26,8 +26,8 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5
+    neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5
     neighbors_dataset: neighbors_filter_4
-    filter_spec: /home/ec2-user/[PATH]/restrictive-filter-test.yml
+    filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-test.yml