From 6af7f2d9fd76f0b01014d346eca3fa44fe46767c Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Wed, 11 Oct 2023 16:37:43 -0700 Subject: [PATCH] Improvement in release test Signed-off-by: Heemin Kim --- .../relaxed-filter/relaxed-filter-test.yml | 11 +- .../restrictive-filter-test.yml | 11 +- .../release-configs/faiss-hnsw/test.yml | 11 +- .../release-configs/faiss-hnswpq/test.yml | 15 +-- .../relaxed-filter/relaxed-filter-test.yml | 17 +-- .../restrictive-filter-test.yml | 17 +-- .../release-configs/faiss-ivf/test.yml | 15 +-- .../release-configs/faiss-ivfpq/test.yml | 15 +-- .../relaxed-filter/relaxed-filter-test.yml | 11 +- .../restrictive-filter-test.yml | 11 +- .../release-configs/lucene-hnsw/test.yml | 9 +- .../release-configs/nmslib-hnsw/test.yml | 9 +- .../release-configs/run_all_tests.sh | 100 ++++++++++++++++++ 13 files changed, 182 insertions(+), 70 deletions(-) create mode 100755 benchmarks/perf-tool/release-configs/run_all_tests.sh diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index ebbdc4fc05..4e72a50f39 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Relaxed Filter Test" test_id: "Faiss HNSW Relaxed Filter Test" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/index.json + index_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -31,9 +32,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/relaxed-filter-spec.json + filter_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index 4cdaf46b05..436815aa4f 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Restrictive Filter Test" test_id: "Faiss HNSW Restrictive Filter Test" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/index.json + index_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -31,9 +32,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/restrictive-filter-spec.json + filter_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml index 5741e89965..34d94f0d6d 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml @@ -1,4 +1,5 @@ -endpoint: localhost +endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Test" test_id: "Faiss HNSW Test" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge @@ -29,6 +30,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml index f67368415f..78cce3ef00 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW PQ Test" test_id: "Faiss HNSW PQ Test" num_runs: 10 @@ -8,13 +9,13 @@ setup: index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-hnswpq/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,17 +29,17 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-hnswpq/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-hnswpq/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge @@ -53,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dateset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml index c498d70fc9..79d0f91069 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss IVF Relaxed Filter Test" test_id: "Faiss IVF Relaxed Filter Test" num_runs: 10 @@ -8,13 +9,13 @@ setup: index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/train-index-spec.json + index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,17 +29,17 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/method-spec.json + method_spec: release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/index.json + index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -55,9 +56,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: /home/ec2-user/[PATH]/filtering/relaxed-filter/relaxed-filter-spec.json + filter_spec: release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml index c75348ab53..178d18cb01 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss IVF restrictive Filter Test" test_id: "Faiss IVF restrictive Filter Test" num_runs: 10 @@ -8,13 +9,13 @@ setup: index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/train-index-spec.json + index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,17 +29,17 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/method-spec.json + method_spec: release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/index.json + index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -55,9 +56,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/restrictive-filter-spec.json + filter_spec: release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml index f50faa8ea7..c375112a0d 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss IVF" test_id: "Faiss IVF" num_runs: 10 @@ -8,13 +9,13 @@ setup: index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-ivf/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/[PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,17 +29,17 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-ivf/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-ivf/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge @@ -53,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml index bd201ea90f..fc59a19564 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss IVF PQ Test" test_id: "Faiss IVF PQ Test" num_runs: 10 @@ -8,13 +9,13 @@ setup: index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-ivfpq/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,17 +29,17 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-ivfpq/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-ivfpq/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge @@ -53,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 49f166bb49..e899dd2a21 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW Relaxed Filter Test" test_id: "Lucene HNSW Relaxed Filter Test" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/index.json + index_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -29,9 +30,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index 26edf85b2d..2fd5709ed0 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW Restrictive Filter Test" test_id: "Lucene HNSW Restrictive Filter Test" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/index.json + index_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -29,9 +30,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: /home/ec2-user/[PATH]/filtering/restrictive-filter/restrictive-filter-spec.json + filter_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml index e11969b760..8876975584 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW" test_id: "Lucene HNSW" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/lucene-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge @@ -27,6 +28,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml index 16fee64520..8d29135a9c 100644 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml @@ -1,4 +1,5 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Nmslib HNSW Test" test_id: "Nmslib HNSW Test" num_runs: 10 @@ -8,13 +9,13 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/nmslib-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge @@ -29,6 +30,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/run_all_tests.sh b/benchmarks/perf-tool/release-configs/run_all_tests.sh new file mode 100755 index 0000000000..5559c47ba7 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/run_all_tests.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -e + +# Description: +# Run a performance test for release +# Dataset should be available in perf-tool/dataset before running this script +# +# Example: +# ./run-test.sh --endpoint localhost +# +# Usage: +# ./run-test.sh \ +# --endpoint +# --port 80 \ +# --num-runs 10 \ +# --outputs ~/outputs + +while [ "$1" != "" ]; do + case $1 in + -url | --endpoint ) shift + ENDPOINT=$1 + ;; + -p | --port ) shift + PORT=$1 + ;; + -n | --num-runs ) shift + NUM_RUNS=$1 + ;; + -o | --outputs ) shift + OUTPUTS=$1 + ;; + * ) echo "Unknown parameter" + echo $1 + exit 1 + ;; + esac + shift +done + +if [ ! -n "$ENDPOINT" ]; then + echo "--endpoint should be specified" + exit +fi + +if [ ! -n "$PORT" ]; then + PORT=80 + echo "--port is not specified. Using default values $PORT" +fi + +if [ ! -n "$NUM_RUNS" ]; then + NUM_RUNS=10 + echo "--num-runs is not specified. Using default values $NUM_RUNS" +fi + +if [ ! -n "$OUTPUTS" ]; then + OUTPUTS="$HOME/outputs" + echo "--outputs is not specified. Using default values $OUTPUTS" +fi + + +curl -X PUT "http://$ENDPOINT:$PORT/_cluster/settings?pretty" -H 'Content-Type: application/json' -d' +{ + "persistent" : { + "knn.algo_param.index_thread_qty" : 4 + } +} +' + +TESTS="./release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/faiss-hnsw/test.yml +./release-configs/faiss-hnswpq/test.yml +./release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/faiss-ivf/test.yml +./release-configs/faiss-ivfpq/test.yml +./release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/lucene-hnsw/test.yml +./release-configs/nmslib-hnsw/test.yml" + +if [ ! -d $OUTPUTS ] +then + mkdir $OUTPUTS +fi + +for TEST in $TESTS +do + ORG_FILE=$TEST + NEW_FILE="$ORG_FILE.tmp" + OUT_FILE=$(grep test_id $ORG_FILE | cut -d':' -f2 | sed -r 's/^ "|"$//g' | sed 's/ /_/g') + echo "cp $ORG_FILE $NEW_FILE" + cp $ORG_FILE $NEW_FILE + sed -i "/^endpoint:/c\endpoint: $ENDPOINT" $NEW_FILE + sed -i "/^port:/c\port: $PORT" $NEW_FILE + sed -i "/^num_runs:/c\num_runs: $NUM_RUNS" $NEW_FILE + python3 knn-perf-tool.py test $NEW_FILE $OUTPUTS/$OUT_FILE + #Sleep for 1 min to cool down cpu from the previous run + sleep 60 +done