Improvement in release test

Signed-off-by: Heemin Kim <[email protected]>
heemin32 · Oct 12, 2023 · 6af7f2d · 6af7f2d
1 parent 34d697f
commit 6af7f2d
Show file tree

Hide file tree

Showing 13 changed files with 182 additions and 70 deletions.
diff --git a/...rks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/...rks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss HNSW Relaxed Filter Test"
 test_id: "Faiss HNSW Relaxed Filter Test"
 num_runs: 10
@@ -8,13 +9,13 @@ steps:
     index_name: target_index
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/index.json
+    index_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/index.json
   - name: ingest_multi_field
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     attributes_dataset_name: attributes
     attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ]
   - name: refresh_index
@@ -31,9 +32,9 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters-updated.hdf5
+    neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5
     neighbors_dataset: neighbors_filter_5
-    filter_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/relaxed-filter-spec.json
+    filter_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json
     filter_type: FILTER
diff --git a/...-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/...-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss HNSW Restrictive Filter Test"
 test_id: "Faiss HNSW Restrictive Filter Test"
 num_runs: 10
@@ -8,13 +9,13 @@ steps:
     index_name: target_index
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/index.json
+    index_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/index.json
   - name: ingest_multi_field
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     attributes_dataset_name: attributes
     attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ]
   - name: refresh_index
@@ -31,9 +32,9 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5
+    neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5
     neighbors_dataset: neighbors_filter_4
-    filter_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/restrictive-filter-spec.json
+    filter_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json
     filter_type: FILTER
diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml
@@ -1,4 +1,5 @@
-endpoint: localhost
+endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss HNSW Test"
 test_id: "Faiss HNSW Test"
 num_runs: 10
@@ -8,13 +9,13 @@ steps:
     index_name: target_index
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/index.json
+    index_spec: release-configs/faiss-hnsw/index.json
   - name: ingest
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
   - name: refresh_index
     index_name: target_index
   - name: force_merge
@@ -29,6 +30,6 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     neighbors_format: hdf5
-    neighbors_path: [DATASET_PATH]/sift-128-euclidean.hdf5
+    neighbors_path: dataset/sift-128-euclidean.hdf5
diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss HNSW PQ Test"
 test_id: "Faiss HNSW PQ Test"
 num_runs: 10
@@ -8,13 +9,13 @@ setup:
     index_name: train_index
   - name: create_index
     index_name: train_index
-    index_spec: /home/ec2-user/[PATH]/train-index-spec.json
+    index_spec: release-configs/faiss-hnswpq/train-index-spec.json
   - name: ingest
     index_name: train_index
     field_name: train_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     doc_count: 50000
   - name: refresh_index
     index_name: train_index
@@ -28,17 +29,17 @@ steps:
     train_index: train_index
     train_field: train_field
     dimension: 128
-    method_spec: /home/ec2-user/[PATH]/method-spec.json
+    method_spec: release-configs/faiss-hnswpq/method-spec.json
     max_training_vector_count: 50000
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/index.json
+    index_spec: release-configs/faiss-hnswpq/index.json
   - name: ingest
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
   - name: refresh_index
     index_name: target_index
   - name: force_merge
@@ -53,6 +54,6 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dateset/sift-128-euclidean.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    neighbors_path: dataset/sift-128-euclidean.hdf5
diff --git a/...arks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml b/...arks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss IVF Relaxed Filter Test"
 test_id: "Faiss IVF Relaxed Filter Test"
 num_runs: 10
@@ -8,13 +9,13 @@ setup:
     index_name: train_index
   - name: create_index
     index_name: train_index
-    index_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/train-index-spec.json
+    index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json
   - name: ingest
     index_name: train_index
     field_name: train_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     doc_count: 50000
   - name: refresh_index
     index_name: train_index
@@ -28,17 +29,17 @@ steps:
     train_index: train_index
     train_field: train_field
     dimension: 128
-    method_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/method-spec.json
+    method_spec: release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json
     max_training_vector_count: 50000
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/index.json
+    index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/index.json
   - name: ingest_multi_field
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     attributes_dataset_name: attributes
     attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ]
   - name: refresh_index
@@ -55,9 +56,9 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters-updated.hdf5
+    neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5
     neighbors_dataset: neighbors_filter_5
-    filter_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/relaxed-filter-spec.json
+    filter_spec: release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json
     filter_type: FILTER
diff --git a/...f-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml b/...f-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss IVF restrictive Filter Test"
 test_id: "Faiss IVF restrictive Filter Test"
 num_runs: 10
@@ -8,13 +9,13 @@ setup:
     index_name: train_index
   - name: create_index
     index_name: train_index
-    index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/train-index-spec.json
+    index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json
   - name: ingest
     index_name: train_index
     field_name: train_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     doc_count: 50000
   - name: refresh_index
     index_name: train_index
@@ -28,17 +29,17 @@ steps:
     train_index: train_index
     train_field: train_field
     dimension: 128
-    method_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/method-spec.json
+    method_spec: release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json
     max_training_vector_count: 50000
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/index.json
+    index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/index.json
   - name: ingest_multi_field
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     attributes_dataset_name: attributes
     attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ]
   - name: refresh_index
@@ -55,9 +56,9 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5
+    dataset_path: dataset/sift-128-euclidean-with-attr.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5
+    neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5
     neighbors_dataset: neighbors_filter_4
-    filter_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/restrictive-filter-spec.json
+    filter_spec: release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json
     filter_type: FILTER
diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss IVF"
 test_id: "Faiss IVF"
 num_runs: 10
@@ -8,13 +9,13 @@ setup:
     index_name: train_index
   - name: create_index
     index_name: train_index
-    index_spec: /home/ec2-user/[PATH]/train-index-spec.json
+    index_spec: release-configs/faiss-ivf/train-index-spec.json
   - name: ingest
     index_name: train_index
     field_name: train_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/[PATH]/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     doc_count: 50000
   - name: refresh_index
     index_name: train_index
@@ -28,17 +29,17 @@ steps:
     train_index: train_index
     train_field: train_field
     dimension: 128
-    method_spec: /home/ec2-user/[PATH]/method-spec.json
+    method_spec: release-configs/faiss-ivf/method-spec.json
     max_training_vector_count: 50000
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/index.json
+    index_spec: release-configs/faiss-ivf/index.json
   - name: ingest
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
   - name: refresh_index
     index_name: target_index
   - name: force_merge
@@ -53,6 +54,6 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    neighbors_path: dataset/sift-128-euclidean.hdf5
diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml
@@ -1,4 +1,5 @@
 endpoint: [ENDPOINT]
+port: [PORT]
 test_name: "Faiss IVF PQ Test"
 test_id: "Faiss IVF PQ Test"
 num_runs: 10
@@ -8,13 +9,13 @@ setup:
     index_name: train_index
   - name: create_index
     index_name: train_index
-    index_spec: /home/ec2-user/[PATH]/train-index-spec.json
+    index_spec: release-configs/faiss-ivfpq/train-index-spec.json
   - name: ingest
     index_name: train_index
     field_name: train_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     doc_count: 50000
   - name: refresh_index
     index_name: train_index
@@ -28,17 +29,17 @@ steps:
     train_index: train_index
     train_field: train_field
     dimension: 128
-    method_spec: /home/ec2-user/[PATH]/method-spec.json
+    method_spec: release-configs/faiss-ivfpq/method-spec.json
     max_training_vector_count: 50000
   - name: create_index
     index_name: target_index
-    index_spec: /home/ec2-user/[PATH]/index.json
+    index_spec: release-configs/faiss-ivfpq/index.json
   - name: ingest
     index_name: target_index
     field_name: target_field
     bulk_size: 500
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
   - name: refresh_index
     index_name: target_index
   - name: force_merge
@@ -53,6 +54,6 @@ steps:
     index_name: target_index
     field_name: target_field
     dataset_format: hdf5
-    dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    dataset_path: dataset/sift-128-euclidean.hdf5
     neighbors_format: hdf5
-    neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5
+    neighbors_path: dataset/sift-128-euclidean.hdf5