From fd8846788532c6bd737eb0563d3913f37113d266 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 24 Oct 2023 15:39:06 -0700 Subject: [PATCH] Full sync benchmark test folder from main to 2.x Signed-off-by: Heemin Kim --- benchmarks/osb/README.md | 485 +++++++++--------- benchmarks/osb/params/no-train-params.json | 2 + benchmarks/osb/params/train-params.json | 2 + benchmarks/osb/procedures/no-train-test.json | 10 + benchmarks/osb/procedures/train-test.json | 10 + .../perf-tool/okpt/io/config/parsers/test.py | 2 + .../perf-tool/okpt/io/config/schemas/test.yml | 3 + .../perf-tool/okpt/test/steps/factory.py | 4 +- benchmarks/perf-tool/okpt/test/steps/steps.py | 41 +- .../filtering/relaxed-filter/index.json | 3 +- .../relaxed-filter/relaxed-filter-test.yml | 18 +- .../filtering/restrictive-filter/index.json | 3 +- .../restrictive-filter-test.yml | 15 +- .../release-configs/faiss-hnsw/index.json | 3 +- .../release-configs/faiss-hnsw/test.yml | 15 +- .../release-configs/faiss-hnswpq/test.yml | 26 +- .../filtering/relaxed-filter/index.json | 17 + .../filtering/relaxed-filter/method-spec.json | 9 + .../relaxed-filter/relaxed-filter-spec.json | 42 ++ .../relaxed-filter/relaxed-filter-test.yml | 64 +++ .../relaxed-filter/train-index-spec.json | 16 + .../filtering/restrictive-filter/index.json | 17 + .../restrictive-filter/method-spec.json | 9 + .../restrictive-filter-spec.json | 44 ++ .../restrictive-filter-test.yml | 64 +++ .../restrictive-filter/train-index-spec.json | 16 + .../release-configs/faiss-ivf/test.yml | 28 +- .../release-configs/faiss-ivfpq/test.yml | 26 +- .../relaxed-filter/relaxed-filter-spec.json | 4 +- .../relaxed-filter/relaxed-filter-test.yml | 16 +- .../restrictive-filter-test.yml | 13 +- .../release-configs/lucene-hnsw/test.yml | 18 +- .../release-configs/nmslib-hnsw/index.json | 2 +- .../release-configs/nmslib-hnsw/test.yml | 20 +- .../release-configs/run_all_tests.sh | 100 ++++ .../sample-configs/faiss-sift-ivf/test.yml | 2 + .../sample-configs/nmslib-sift-hnsw/test.yml | 2 + 37 files changed, 849 insertions(+), 322 deletions(-) create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json create mode 100755 benchmarks/perf-tool/release-configs/run_all_tests.sh diff --git a/benchmarks/osb/README.md b/benchmarks/osb/README.md index 92272e20b..0f806a344 100644 --- a/benchmarks/osb/README.md +++ b/benchmarks/osb/README.md @@ -106,26 +106,28 @@ use an algorithm that requires training. #### Parameters -| Name | Description | -|-----------------------------------------|----------------------------------------------------------------------------------| -| target_index_name | Name of index to add vectors to | -| target_field_name | Name of field to add vectors to | -| target_index_body | Path to target index definition | -| target_index_primary_shards | Target index primary shards | -| target_index_replica_shards | Target index replica shards | -| target_index_dimension | Dimension of target index | -| target_index_space_type | Target index space type | -| target_index_bulk_size | Target index bulk size | -| target_index_bulk_index_data_set_format | Format of vector data set | -| target_index_bulk_index_data_set_path | Path to vector data set | -| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| hnsw_ef_search | HNSW ef search parameter | -| hnsw_ef_construction | HNSW ef construction parameter | -| hnsw_m | HNSW m parameter | -| query_k | The number of neighbors to return for the search | -| query_clients | Number of clients to use for running queries | -| query_data_set_format | Format of vector data set for queries | -| query_data_set_path | Path to vector data set for queries | +| Name | Description | +|-----------------------------------------|--------------------------------------------------------------------------| +| target_index_name | Name of index to add vectors to | +| target_field_name | Name of field to add vectors to | +| target_index_body | Path to target index definition | +| target_index_primary_shards | Target index primary shards | +| target_index_replica_shards | Target index replica shards | +| target_index_dimension | Dimension of target index | +| target_index_space_type | Target index space type | +| target_index_bulk_size | Target index bulk size | +| target_index_bulk_index_data_set_format | Format of vector data set | +| target_index_bulk_index_data_set_path | Path to vector data set | +| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | +| target_index_max_num_segments | Number of segments to merge target index down to before beginning search | +| target_index_force_merge_timeout | Timeout for of force merge requests in seconds | +| hnsw_ef_search | HNSW ef search parameter | +| hnsw_ef_construction | HNSW ef construction parameter | +| hnsw_m | HNSW m parameter | +| query_k | The number of neighbors to return for the search | +| query_clients | Number of clients to use for running queries | +| query_data_set_format | Format of vector data set for queries | +| query_data_set_path | Path to vector data set for queries | #### Metrics @@ -141,86 +143,90 @@ The result metrics of this procedure will look like: | Metric | Task | Value | Unit | |---------------------------------------------------------------:|------------------------:|------------:|-------:| -| Cumulative indexing time of primary shards | | 0.00173333 | min | -| Min cumulative indexing time across primary shards | | 0 | min | -| Median cumulative indexing time across primary shards | | 0 | min | -| Max cumulative indexing time across primary shards | | 0.000616667 | min | +| Cumulative indexing time of primary shards | | 1.82885 | min | +| Min cumulative indexing time across primary shards | | 0.4121 | min | +| Median cumulative indexing time across primary shards | | 0.559617 | min | +| Max cumulative indexing time across primary shards | | 0.857133 | min | | Cumulative indexing throttle time of primary shards | | 0 | min | | Min cumulative indexing throttle time across primary shards | | 0 | min | | Median cumulative indexing throttle time across primary shards | | 0 | min | | Max cumulative indexing throttle time across primary shards | | 0 | min | -| Cumulative merge time of primary shards | | 0 | min | -| Cumulative merge count of primary shards | | 0 | | -| Min cumulative merge time across primary shards | | 0 | min | -| Median cumulative merge time across primary shards | | 0 | min | -| Max cumulative merge time across primary shards | | 0 | min | +| Cumulative merge time of primary shards | | 5.89065 | min | +| Cumulative merge count of primary shards | | 3 | | +| Min cumulative merge time across primary shards | | 1.95945 | min | +| Median cumulative merge time across primary shards | | 1.96345 | min | +| Max cumulative merge time across primary shards | | 1.96775 | min | | Cumulative merge throttle time of primary shards | | 0 | min | | Min cumulative merge throttle time across primary shards | | 0 | min | | Median cumulative merge throttle time across primary shards | | 0 | min | | Max cumulative merge throttle time across primary shards | | 0 | min | -| Cumulative refresh time of primary shards | | 0.00271667 | min | -| Cumulative refresh count of primary shards | | 115 | | -| Min cumulative refresh time across primary shards | | 0 | min | -| Median cumulative refresh time across primary shards | | 0 | min | -| Max cumulative refresh time across primary shards | | 0.00135 | min | -| Cumulative flush time of primary shards | | 0 | min | -| Cumulative flush count of primary shards | | 43 | | -| Min cumulative flush time across primary shards | | 0 | min | -| Median cumulative flush time across primary shards | | 0 | min | -| Max cumulative flush time across primary shards | | 0 | min | -| Total Young Gen GC time | | 0.849 | s | -| Total Young Gen GC count | | 20 | | +| Cumulative refresh time of primary shards | | 8.52517 | min | +| Cumulative refresh count of primary shards | | 29 | | +| Min cumulative refresh time across primary shards | | 2.64265 | min | +| Median cumulative refresh time across primary shards | | 2.93913 | min | +| Max cumulative refresh time across primary shards | | 2.94338 | min | +| Cumulative flush time of primary shards | | 0.00221667 | min | +| Cumulative flush count of primary shards | | 3 | | +| Min cumulative flush time across primary shards | | 0.000733333 | min | +| Median cumulative flush time across primary shards | | 0.000733333 | min | +| Max cumulative flush time across primary shards | | 0.00075 | min | +| Total Young Gen GC time | | 0.318 | s | +| Total Young Gen GC count | | 2 | | | Total Old Gen GC time | | 0 | s | | Total Old Gen GC count | | 0 | | -| Store size | | 0.647921 | GB | -| Translog size | | 0.00247511 | GB | -| Heap used for segments | | 0.284451 | MB | -| Heap used for doc values | | 0.0872688 | MB | -| Heap used for terms | | 0.0714417 | MB | -| Heap used for norms | | 6.10352e-05 | MB | +| Store size | | 1.43566 | GB | +| Translog size | | 1.53668e-07 | GB | +| Heap used for segments | | 0.00410843 | MB | +| Heap used for doc values | | 0.000286102 | MB | +| Heap used for terms | | 0.00121307 | MB | +| Heap used for norms | | 0 | MB | | Heap used for points | | 0 | MB | -| Heap used for stored fields | | 0.125679 | MB | -| Segment count | | 257 | | -| Min Throughput | custom-vector-bulk | 18018.5 | docs/s | -| Mean Throughput | custom-vector-bulk | 18018.5 | docs/s | -| Median Throughput | custom-vector-bulk | 18018.5 | docs/s | -| Max Throughput | custom-vector-bulk | 18018.5 | docs/s | -| 50th percentile latency | custom-vector-bulk | 98.5565 | ms | -| 90th percentile latency | custom-vector-bulk | 100.033 | ms | -| 100th percentile latency | custom-vector-bulk | 103.792 | ms | -| 50th percentile service time | custom-vector-bulk | 98.5565 | ms | -| 90th percentile service time | custom-vector-bulk | 100.033 | ms | -| 100th percentile service time | custom-vector-bulk | 103.792 | ms | +| Heap used for stored fields | | 0.00260925 | MB | +| Segment count | | 3 | | +| Min Throughput | custom-vector-bulk | 38005.8 | docs/s | +| Mean Throughput | custom-vector-bulk | 44827.9 | docs/s | +| Median Throughput | custom-vector-bulk | 40507.2 | docs/s | +| Max Throughput | custom-vector-bulk | 88967.8 | docs/s | +| 50th percentile latency | custom-vector-bulk | 29.5857 | ms | +| 90th percentile latency | custom-vector-bulk | 49.0719 | ms | +| 99th percentile latency | custom-vector-bulk | 72.6138 | ms | +| 99.9th percentile latency | custom-vector-bulk | 279.826 | ms | +| 100th percentile latency | custom-vector-bulk | 15688 | ms | +| 50th percentile service time | custom-vector-bulk | 29.5857 | ms | +| 90th percentile service time | custom-vector-bulk | 49.0719 | ms | +| 99th percentile service time | custom-vector-bulk | 72.6138 | ms | +| 99.9th percentile service time | custom-vector-bulk | 279.826 | ms | +| 100th percentile service time | custom-vector-bulk | 15688 | ms | | error rate | custom-vector-bulk | 0 | % | -| Min Throughput | refresh-target-index | 76.22 | ops/s | -| Mean Throughput | refresh-target-index | 76.22 | ops/s | -| Median Throughput | refresh-target-index | 76.22 | ops/s | -| Max Throughput | refresh-target-index | 76.22 | ops/s | -| 100th percentile latency | refresh-target-index | 12.7619 | ms | -| 100th percentile service time | refresh-target-index | 12.7619 | ms | +| Min Throughput | refresh-target-index | 0.01 | ops/s | +| Mean Throughput | refresh-target-index | 0.01 | ops/s | +| Median Throughput | refresh-target-index | 0.01 | ops/s | +| Max Throughput | refresh-target-index | 0.01 | ops/s | +| 100th percentile latency | refresh-target-index | 176610 | ms | +| 100th percentile service time | refresh-target-index | 176610 | ms | | error rate | refresh-target-index | 0 | % | -| Min Throughput | knn-query-from-data-set | 1587.47 | ops/s | -| Mean Throughput | knn-query-from-data-set | 1649.97 | ops/s | -| Median Throughput | knn-query-from-data-set | 1661.79 | ops/s | -| Max Throughput | knn-query-from-data-set | 1677.06 | ops/s | -| 50th percentile latency | knn-query-from-data-set | 4.79125 | ms | -| 90th percentile latency | knn-query-from-data-set | 5.38 | ms | -| 99th percentile latency | knn-query-from-data-set | 46.8965 | ms | -| 99.9th percentile latency | knn-query-from-data-set | 58.2049 | ms | -| 99.99th percentile latency | knn-query-from-data-set | 59.6476 | ms | -| 100th percentile latency | knn-query-from-data-set | 60.9245 | ms | -| 50th percentile service time | knn-query-from-data-set | 4.79125 | ms | -| 90th percentile service time | knn-query-from-data-set | 5.38 | ms | -| 99th percentile service time | knn-query-from-data-set | 46.8965 | ms | -| 99.9th percentile service time | knn-query-from-data-set | 58.2049 | ms | -| 99.99th percentile service time | knn-query-from-data-set | 59.6476 | ms | -| 100th percentile service time | knn-query-from-data-set | 60.9245 | ms | +| Min Throughput | knn-query-from-data-set | 444.17 | ops/s | +| Mean Throughput | knn-query-from-data-set | 601.68 | ops/s | +| Median Throughput | knn-query-from-data-set | 621.19 | ops/s | +| Max Throughput | knn-query-from-data-set | 631.23 | ops/s | +| 50th percentile latency | knn-query-from-data-set | 14.7612 | ms | +| 90th percentile latency | knn-query-from-data-set | 20.6954 | ms | +| 99th percentile latency | knn-query-from-data-set | 27.7499 | ms | +| 99.9th percentile latency | knn-query-from-data-set | 41.3506 | ms | +| 99.99th percentile latency | knn-query-from-data-set | 162.391 | ms | +| 100th percentile latency | knn-query-from-data-set | 162.756 | ms | +| 50th percentile service time | knn-query-from-data-set | 14.7612 | ms | +| 90th percentile service time | knn-query-from-data-set | 20.6954 | ms | +| 99th percentile service time | knn-query-from-data-set | 27.7499 | ms | +| 99.9th percentile service time | knn-query-from-data-set | 41.3506 | ms | +| 99.99th percentile service time | knn-query-from-data-set | 162.391 | ms | +| 100th percentile service time | knn-query-from-data-set | 162.756 | ms | | error rate | knn-query-from-data-set | 0 | % | --------------------------------- -[INFO] SUCCESS (took 46 seconds) --------------------------------- +--------------------------------- +[INFO] SUCCESS (took 618 seconds) +--------------------------------- ``` ### Train Test @@ -243,41 +249,43 @@ algorithm that requires training. #### Parameters -| Name | Description | -|-----------------------------------------|----------------------------------------------------------------------------------| -| target_index_name | Name of index to add vectors to | -| target_field_name | Name of field to add vectors to | -| target_index_body | Path to target index definition | -| target_index_primary_shards | Target index primary shards | -| target_index_replica_shards | Target index replica shards | -| target_index_dimension | Dimension of target index | -| target_index_space_type | Target index space type | -| target_index_bulk_size | Target index bulk size | -| target_index_bulk_index_data_set_format | Format of vector data set for ingestion | -| target_index_bulk_index_data_set_path | Path to vector data set for ingestion | -| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| ivf_nlists | IVF nlist parameter | -| ivf_nprobes | IVF nprobe parameter | -| pq_code_size | PQ code_size parameter | -| pq_m | PQ m parameter | -| train_model_method | Method to be used for model (ivf or ivfpq) | -| train_model_id | Model ID | -| train_index_name | Name of index to put training data into | -| train_field_name | Name of field to put training data into | -| train_index_body | Path to train index definition | -| train_search_size | Search size to use when pulling training data | -| train_timeout | Timeout to wait for training to finish | -| train_index_primary_shards | Train index primary shards | -| train_index_replica_shards | Train index replica shards | -| train_index_bulk_size | Train index bulk size | -| train_index_data_set_format | Format of vector data set for training | -| train_index_data_set_path | Path to vector data set for training | -| train_index_num_vectors | Number of vectors to use from vector data set for training | -| train_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| query_k | The number of neighbors to return for the search | -| query_clients | Number of clients to use for running queries | -| query_data_set_format | Format of vector data set for queries | -| query_data_set_path | Path to vector data set for queries | +| Name | Description | +|-----------------------------------------|--------------------------------------------------------------------------| +| target_index_name | Name of index to add vectors to | +| target_field_name | Name of field to add vectors to | +| target_index_body | Path to target index definition | +| target_index_primary_shards | Target index primary shards | +| target_index_replica_shards | Target index replica shards | +| target_index_dimension | Dimension of target index | +| target_index_space_type | Target index space type | +| target_index_bulk_size | Target index bulk size | +| target_index_bulk_index_data_set_format | Format of vector data set for ingestion | +| target_index_bulk_index_data_set_path | Path to vector data set for ingestion | +| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | +| target_index_max_num_segments | Number of segments to merge target index down to before beginning search | +| target_index_force_merge_timeout | Timeout for of force merge requests in seconds | +| ivf_nlists | IVF nlist parameter | +| ivf_nprobes | IVF nprobe parameter | +| pq_code_size | PQ code_size parameter | +| pq_m | PQ m parameter | +| train_model_method | Method to be used for model (ivf or ivfpq) | +| train_model_id | Model ID | +| train_index_name | Name of index to put training data into | +| train_field_name | Name of field to put training data into | +| train_index_body | Path to train index definition | +| train_search_size | Search size to use when pulling training data | +| train_timeout | Timeout to wait for training to finish | +| train_index_primary_shards | Train index primary shards | +| train_index_replica_shards | Train index replica shards | +| train_index_bulk_size | Train index bulk size | +| train_index_data_set_format | Format of vector data set for training | +| train_index_data_set_path | Path to vector data set for training | +| train_index_num_vectors | Number of vectors to use from vector data set for training | +| train_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | +| query_k | The number of neighbors to return for the search | +| query_clients | Number of clients to use for running queries | +| query_data_set_format | Format of vector data set for queries | +| query_data_set_path | Path to vector data set for queries | #### Metrics @@ -286,131 +294,132 @@ The result metrics of this procedure will look like: ------------------------------------------------------ _______ __ _____ / ____(_)___ ____ _/ / / ___/_________ ________ - / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ [63/1855] + / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/ /_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ------------------------------------------------------ -| Metric | Task | Value | Unit | -|---------------------------------------------------------------:|------------------------:|------------:|-----------------:| -| Cumulative indexing time of primary shards | | 2.92355 | min | -| Min cumulative indexing time across primary shards | | 0 | min | -| Median cumulative indexing time across primary shards | | 0.497817 | min | -| Max cumulative indexing time across primary shards | | 1.37717 | min | -| Cumulative indexing throttle time of primary shards | | 0 | min | -| Min cumulative indexing throttle time across primary shards | | 0 | min | -| Median cumulative indexing throttle time across primary shards | | 0 | min | -| Max cumulative indexing throttle time across primary shards | | 0 | min | -| Cumulative merge time of primary shards | | 1.34895 | min | -| Cumulative merge count of primary shards | | 39 | | -| Min cumulative merge time across primary shards | | 0 | min | -| Median cumulative merge time across primary shards | | 0.292033 | min | -| Max cumulative merge time across primary shards | | 0.6268 | min | -| Cumulative merge throttle time of primary shards | | 0.62845 | min | -| Min cumulative merge throttle time across primary shards | | 0 | min | -| Median cumulative merge throttle time across primary shards | | 0.155617 | min | -| Max cumulative merge throttle time across primary shards | | 0.290117 | min | -| Cumulative refresh time of primary shards | | 0.369433 | min | -| Cumulative refresh count of primary shards | | 96 | | -| Min cumulative refresh time across primary shards | | 0 | min | -| Median cumulative refresh time across primary shards | | 0.0903833 | min | -| Max cumulative refresh time across primary shards | | 0.10365 | min | -| Cumulative flush time of primary shards | | 0.0278667 | min | -| Cumulative flush count of primary shards | | 2 | | -| Min cumulative flush time across primary shards | | 0 | min | -| Median cumulative flush time across primary shards | | 0 | min | -| Max cumulative flush time across primary shards | | 0.0278667 | min | -| Total Young Gen GC time | | 13.106 | s | -| Total Young Gen GC count | | 263 | | -| Total Old Gen GC time | | 0 | s | -| Total Old Gen GC count | | 0 | | -| Store size | | 2.60183 | GB | -| Translog size | | 1.34787 | GB | -| Heap used for segments | | 0.0646248 | MB | -| Heap used for doc values | | 0.00899887 | MB | -| Heap used for terms | | 0.0203552 | MB | -| Heap used for norms | | 6.10352e-05 | MB | -| Heap used for points | | 0 | MB | -| Heap used for stored fields | | 0.0352097 | MB | -| Segment count | | 71 | | -| Min Throughput | delete-model | 10.55 | ops/s | -| Mean Throughput | delete-model | 10.55 | ops/s | -| Median Throughput | delete-model | 10.55 | ops/s | -| Max Throughput | delete-model | 10.55 | ops/s | -| 100th percentile latency | delete-model | 94.4726 | ms | -| 100th percentile service time | delete-model | 94.4726 | ms | -| error rate | delete-model | 0 | % | -| Min Throughput | train-vector-bulk | 44763.1 | docs/s | -| Mean Throughput | train-vector-bulk | 52022.4 | docs/s | -| Median Throughput | train-vector-bulk | 52564.8 | docs/s | -| Max Throughput | train-vector-bulk | 53833 | docs/s | -| 50th percentile latency | train-vector-bulk | 22.3364 | ms | -| 90th percentile latency | train-vector-bulk | 47.799 | ms | -| 99th percentile latency | train-vector-bulk | 195.954 | ms | -| 99.9th percentile latency | train-vector-bulk | 495.217 | ms | -| 100th percentile latency | train-vector-bulk | 663.48 | ms | -| 50th percentile service time | train-vector-bulk | 22.3364 | ms | -| 90th percentile service time | train-vector-bulk | 47.799 | ms | -| 99th percentile service time | train-vector-bulk | 195.954 | ms | -| 99.9th percentile service time | train-vector-bulk | 495.217 | ms | -| 100th percentile service time | train-vector-bulk | 663.48 | ms | -| error rate | train-vector-bulk | 0 | % | -| Min Throughput | refresh-train-index | 0.98 | ops/s | -| Mean Throughput | refresh-train-index | 0.98 | ops/s | -| Median Throughput | refresh-train-index | 0.98 | ops/s | -| Max Throughput | refresh-train-index | 0.98 | ops/s | -| 100th percentile latency | refresh-train-index | 1019.54 | ms | -| 100th percentile service time | refresh-train-index | 1019.54 | ms | -| error rate | refresh-train-index | 0 | % | -| Min Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Mean Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Median Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Max Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| 100th percentile latency | ivfpq-train-model | 150952 | ms | -| 100th percentile service time | ivfpq-train-model | 150952 | ms | -| error rate | ivfpq-train-model | 0 | % | -| Min Throughput | custom-vector-bulk | 32367.4 | docs/s | -| Mean Throughput | custom-vector-bulk | 36027.5 | docs/s | -| Median Throughput | custom-vector-bulk | 35276.7 | docs/s | -| Max Throughput | custom-vector-bulk | 41095 | docs/s | -| 50th percentile latency | custom-vector-bulk | 22.2419 | ms | -| 90th percentile latency | custom-vector-bulk | 70.163 | ms | -| 99th percentile latency | custom-vector-bulk | 308.395 | ms | -| 99.9th percentile latency | custom-vector-bulk | 548.558 | ms | -| 100th percentile latency | custom-vector-bulk | 655.628 | ms | -| 50th percentile service time | custom-vector-bulk | 22.2419 | ms | -| 90th percentile service time | custom-vector-bulk | 70.163 | ms | -| 99th percentile service time | custom-vector-bulk | 308.395 | ms | -| 99.9th percentile service time | custom-vector-bulk | 548.558 | ms | -| 100th percentile service time | custom-vector-bulk | 655.628 | ms | -| error rate | custom-vector-bulk | 0 | % | -| Min Throughput | refresh-target-index | 0.23 | ops/s | -| Mean Throughput | refresh-target-index | 0.23 | ops/s | -| Median Throughput | refresh-target-index | 0.23 | ops/s | -| Max Throughput | refresh-target-index | 0.23 | ops/s | -| 100th percentile latency | refresh-target-index | 4331.17 | ms | -| 100th percentile service time | refresh-target-index | 4331.17 | ms | -| error rate | refresh-target-index | 0 | % | -| Min Throughput | knn-query-from-data-set | 455.19 | ops/s | -| Mean Throughput | knn-query-from-data-set | 511.74 | ops/s | -| Median Throughput | knn-query-from-data-set | 510.85 | ops/s | -| Max Throughput | knn-query-from-data-set | 570.07 | ops/s | -| 50th percentile latency | knn-query-from-data-set | 14.1626 | ms | -| 90th percentile latency | knn-query-from-data-set | 30.2389 | ms | -| 99th percentile latency | knn-query-from-data-set | 71.2793 | ms | -| 99.9th percentile latency | knn-query-from-data-set | 104.733 | ms | -| 99.99th percentile latency | knn-query-from-data-set | 127.298 | ms | -| 100th percentile latency | knn-query-from-data-set | 145.229 | ms | -| 50th percentile service time | knn-query-from-data-set | 14.1626 | ms | -| 90th percentile service time | knn-query-from-data-set | 30.2389 | ms | -| 99th percentile service time | knn-query-from-data-set | 71.2793 | ms | -| 99.9th percentile service time | knn-query-from-data-set | 104.733 | ms | -| 99.99th percentile service time | knn-query-from-data-set | 127.298 | ms | -| 100th percentile service time | knn-query-from-data-set | 145.229 | ms | -| error rate | knn-query-from-data-set | 0 | % | + +| Metric | Task | Value | Unit | +|---------------------------------------------------------------:|------------------------:|-----------:|-----------------:| +| Cumulative indexing time of primary shards | | 2.92382 | min | +| Min cumulative indexing time across primary shards | | 0.42245 | min | +| Median cumulative indexing time across primary shards | | 0.43395 | min | +| Max cumulative indexing time across primary shards | | 1.63347 | min | +| Cumulative indexing throttle time of primary shards | | 0 | min | +| Min cumulative indexing throttle time across primary shards | | 0 | min | +| Median cumulative indexing throttle time across primary shards | | 0 | min | +| Max cumulative indexing throttle time across primary shards | | 0 | min | +| Cumulative merge time of primary shards | | 1.36293 | min | +| Cumulative merge count of primary shards | | 20 | | +| Min cumulative merge time across primary shards | | 0.263283 | min | +| Median cumulative merge time across primary shards | | 0.291733 | min | +| Max cumulative merge time across primary shards | | 0.516183 | min | +| Cumulative merge throttle time of primary shards | | 0.701683 | min | +| Min cumulative merge throttle time across primary shards | | 0.163883 | min | +| Median cumulative merge throttle time across primary shards | | 0.175717 | min | +| Max cumulative merge throttle time across primary shards | | 0.186367 | min | +| Cumulative refresh time of primary shards | | 0.222217 | min | +| Cumulative refresh count of primary shards | | 67 | | +| Min cumulative refresh time across primary shards | | 0.03915 | min | +| Median cumulative refresh time across primary shards | | 0.039825 | min | +| Max cumulative refresh time across primary shards | | 0.103417 | min | +| Cumulative flush time of primary shards | | 0.0276833 | min | +| Cumulative flush count of primary shards | | 1 | | +| Min cumulative flush time across primary shards | | 0 | min | +| Median cumulative flush time across primary shards | | 0 | min | +| Max cumulative flush time across primary shards | | 0.0276833 | min | +| Total Young Gen GC time | | 0.074 | s | +| Total Young Gen GC count | | 8 | | +| Total Old Gen GC time | | 0 | s | +| Total Old Gen GC count | | 0 | | +| Store size | | 1.67839 | GB | +| Translog size | | 0.115145 | GB | +| Heap used for segments | | 0.0350914 | MB | +| Heap used for doc values | | 0.00771713 | MB | +| Heap used for terms | | 0.0101089 | MB | +| Heap used for norms | | 0 | MB | +| Heap used for points | | 0 | MB | +| Heap used for stored fields | | 0.0172653 | MB | +| Segment count | | 25 | | +| Min Throughput | delete-model | 25.45 | ops/s | +| Mean Throughput | delete-model | 25.45 | ops/s | +| Median Throughput | delete-model | 25.45 | ops/s | +| Max Throughput | delete-model | 25.45 | ops/s | +| 100th percentile latency | delete-model | 39.0409 | ms | +| 100th percentile service time | delete-model | 39.0409 | ms | +| error rate | delete-model | 0 | % | +| Min Throughput | train-vector-bulk | 49518.9 | docs/s | +| Mean Throughput | train-vector-bulk | 54418.8 | docs/s | +| Median Throughput | train-vector-bulk | 52984.2 | docs/s | +| Max Throughput | train-vector-bulk | 62118.3 | docs/s | +| 50th percentile latency | train-vector-bulk | 26.5293 | ms | +| 90th percentile latency | train-vector-bulk | 41.8212 | ms | +| 99th percentile latency | train-vector-bulk | 239.351 | ms | +| 99.9th percentile latency | train-vector-bulk | 348.507 | ms | +| 100th percentile latency | train-vector-bulk | 436.292 | ms | +| 50th percentile service time | train-vector-bulk | 26.5293 | ms | +| 90th percentile service time | train-vector-bulk | 41.8212 | ms | +| 99th percentile service time | train-vector-bulk | 239.351 | ms | +| 99.9th percentile service time | train-vector-bulk | 348.507 | ms | +| 100th percentile service time | train-vector-bulk | 436.292 | ms | +| error rate | train-vector-bulk | 0 | % | +| Min Throughput | refresh-train-index | 0.47 | ops/s | +| Mean Throughput | refresh-train-index | 0.47 | ops/s | +| Median Throughput | refresh-train-index | 0.47 | ops/s | +| Max Throughput | refresh-train-index | 0.47 | ops/s | +| 100th percentile latency | refresh-train-index | 2142.96 | ms | +| 100th percentile service time | refresh-train-index | 2142.96 | ms | +| error rate | refresh-train-index | 0 | % | +| Min Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| Mean Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| Median Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| Max Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| 100th percentile latency | ivfpq-train-model | 136563 | ms | +| 100th percentile service time | ivfpq-train-model | 136563 | ms | +| error rate | ivfpq-train-model | 0 | % | +| Min Throughput | custom-vector-bulk | 62384.8 | docs/s | +| Mean Throughput | custom-vector-bulk | 69035.2 | docs/s | +| Median Throughput | custom-vector-bulk | 68675.4 | docs/s | +| Max Throughput | custom-vector-bulk | 80713.4 | docs/s | +| 50th percentile latency | custom-vector-bulk | 18.7726 | ms | +| 90th percentile latency | custom-vector-bulk | 34.8881 | ms | +| 99th percentile latency | custom-vector-bulk | 150.435 | ms | +| 99.9th percentile latency | custom-vector-bulk | 296.862 | ms | +| 100th percentile latency | custom-vector-bulk | 344.394 | ms | +| 50th percentile service time | custom-vector-bulk | 18.7726 | ms | +| 90th percentile service time | custom-vector-bulk | 34.8881 | ms | +| 99th percentile service time | custom-vector-bulk | 150.435 | ms | +| 99.9th percentile service time | custom-vector-bulk | 296.862 | ms | +| 100th percentile service time | custom-vector-bulk | 344.394 | ms | +| error rate | custom-vector-bulk | 0 | % | +| Min Throughput | refresh-target-index | 28.32 | ops/s | +| Mean Throughput | refresh-target-index | 28.32 | ops/s | +| Median Throughput | refresh-target-index | 28.32 | ops/s | +| Max Throughput | refresh-target-index | 28.32 | ops/s | +| 100th percentile latency | refresh-target-index | 34.9811 | ms | +| 100th percentile service time | refresh-target-index | 34.9811 | ms | +| error rate | refresh-target-index | 0 | % | +| Min Throughput | knn-query-from-data-set | 0.9 | ops/s | +| Mean Throughput | knn-query-from-data-set | 453.84 | ops/s | +| Median Throughput | knn-query-from-data-set | 554.15 | ops/s | +| Max Throughput | knn-query-from-data-set | 681 | ops/s | +| 50th percentile latency | knn-query-from-data-set | 11.7174 | ms | +| 90th percentile latency | knn-query-from-data-set | 15.4445 | ms | +| 99th percentile latency | knn-query-from-data-set | 21.0682 | ms | +| 99.9th percentile latency | knn-query-from-data-set | 39.5414 | ms | +| 99.99th percentile latency | knn-query-from-data-set | 1116.33 | ms | +| 100th percentile latency | knn-query-from-data-set | 1116.66 | ms | +| 50th percentile service time | knn-query-from-data-set | 11.7174 | ms | +| 90th percentile service time | knn-query-from-data-set | 15.4445 | ms | +| 99th percentile service time | knn-query-from-data-set | 21.0682 | ms | +| 99.9th percentile service time | knn-query-from-data-set | 39.5414 | ms | +| 99.99th percentile service time | knn-query-from-data-set | 1116.33 | ms | +| 100th percentile service time | knn-query-from-data-set | 1116.66 | ms | +| error rate | knn-query-from-data-set | 0 | % | --------------------------------- -[INFO] SUCCESS (took 295 seconds) +[INFO] SUCCESS (took 281 seconds) --------------------------------- ``` diff --git a/benchmarks/osb/params/no-train-params.json b/benchmarks/osb/params/no-train-params.json index 64fe3c296..58e4197fd 100644 --- a/benchmarks/osb/params/no-train-params.json +++ b/benchmarks/osb/params/no-train-params.json @@ -10,6 +10,8 @@ "target_index_bulk_index_data_set_format": "hdf5", "target_index_bulk_index_data_set_path": "", "target_index_bulk_index_clients": 10, + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 45.0, "hnsw_ef_search": 512, "hnsw_ef_construction": 512, "hnsw_m": 16, diff --git a/benchmarks/osb/params/train-params.json b/benchmarks/osb/params/train-params.json index 4c598d25b..f55ed4333 100644 --- a/benchmarks/osb/params/train-params.json +++ b/benchmarks/osb/params/train-params.json @@ -10,6 +10,8 @@ "target_index_bulk_index_data_set_format": "hdf5", "target_index_bulk_index_data_set_path": "", "target_index_bulk_index_clients": 10, + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 45.0, "ivf_nlists": 10, "ivf_nprobes": 1, "pq_code_size": 8, diff --git a/benchmarks/osb/procedures/no-train-test.json b/benchmarks/osb/procedures/no-train-test.json index 03d72d6bd..01985b914 100644 --- a/benchmarks/osb/procedures/no-train-test.json +++ b/benchmarks/osb/procedures/no-train-test.json @@ -46,6 +46,16 @@ "retries": 100 } }, + { + "operation": { + "name": "force-merge", + "operation-type": "force-merge", + "request-timeout": {{ target_index_force_merge_timeout }}, + "index": "{{ target_index_name }}", + "mode": "polling", + "max-num-segments": {{ target_index_max_num_segments }} + } + }, { "operation": { "name": "knn-query-from-data-set", diff --git a/benchmarks/osb/procedures/train-test.json b/benchmarks/osb/procedures/train-test.json index 49930044a..ca26db0b0 100644 --- a/benchmarks/osb/procedures/train-test.json +++ b/benchmarks/osb/procedures/train-test.json @@ -100,6 +100,16 @@ "retries": 100 } }, + { + "operation": { + "name": "force-merge", + "operation-type": "force-merge", + "request-timeout": {{ target_index_force_merge_timeout }}, + "index": "{{ target_index_name }}", + "mode": "polling", + "max-num-segments": {{ target_index_max_num_segments }} + } + }, { "operation": { "name": "knn-query-from-data-set", diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/test.py b/benchmarks/perf-tool/okpt/io/config/parsers/test.py index d0ef4c02f..c47e30ecc 100644 --- a/benchmarks/perf-tool/okpt/io/config/parsers/test.py +++ b/benchmarks/perf-tool/okpt/io/config/parsers/test.py @@ -24,6 +24,7 @@ class TestConfig: test_id: str endpoint: str port: int + timeout: int num_runs: int show_runs: bool setup: List[Step] @@ -67,6 +68,7 @@ def parse(self, file_obj: TextIOWrapper) -> TestConfig: test_config = TestConfig( endpoint=config_obj['endpoint'], port=config_obj['port'], + timeout=config_obj['timeout'], test_name=config_obj['test_name'], test_id=config_obj['test_id'], num_runs=config_obj['num_runs'], diff --git a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml index 06b880cc7..4d5c21a15 100644 --- a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml +++ b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml @@ -12,6 +12,9 @@ endpoint: port: type: integer default: 9200 +timeout: + type: integer + default: 60 test_name: type: string test_id: diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py index ba0fc5b60..77e963921 100644 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ b/benchmarks/perf-tool/okpt/test/steps/factory.py @@ -10,7 +10,7 @@ from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, \ - QueryStep, QueryWithFilterStep, GetStatsStep + QueryStep, QueryWithFilterStep, GetStatsStep, WarmupStep def create_step(step_config: StepConfig) -> Step: @@ -40,5 +40,7 @@ def create_step(step_config: StepConfig) -> Step: return ClearCacheStep(step_config) elif step_config.step_name == GetStatsStep.label: return GetStatsStep(step_config) + elif step_config.step_name == WarmupStep.label: + return WarmupStep(step_config) raise ConfigurationError(f'Invalid step {step_config.step_name}') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index cc1773330..6ec164d9d 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -38,8 +38,9 @@ def __init__(self, step_config: StepConfig): default_port = 9200 if self.endpoint == 'localhost' else 80 self.port = parse_int_param('port', step_config.config, step_config.implicit_config, default_port) + self.timeout = parse_int_param('timeout', step_config.config, {}, 60) self.opensearch = get_opensearch_client(str(self.endpoint), - int(self.port)) + int(self.port), int(self.timeout)) class CreateIndexStep(OpenSearchStep): @@ -163,6 +164,25 @@ def _get_measures(self) -> List[str]: return ['took'] +class WarmupStep(OpenSearchStep): + """See base class.""" + + label = 'warmup_operation' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + self.index_name = parse_string_param('index_name', step_config.config, {}, + None) + + def _action(self): + """Performs warmup operation on an index.""" + warmup_operation(self.endpoint, self.port, self.index_name) + return {} + + def _get_measures(self) -> List[str]: + return ['took'] + + class TrainModelStep(OpenSearchStep): """See base class.""" @@ -690,6 +710,23 @@ def delete_model(endpoint, port, model_id): return response.json() +def warmup_operation(endpoint, port, index): + """ + Performs warmup operation on index to load native library files + of that index to reduce query latencies. + Args: + endpoint: Endpoint OpenSearch is running on + port: Port OpenSearch is running on + index: index name + Returns: + number of shards the plugin succeeded and failed to warm up. + """ + response = requests.get('http://' + endpoint + ':' + str(port) + + '/_plugins/_knn/warmup/' + index, + headers={'content-type': 'application/json'}) + return response.json() + + def get_opensearch_client(endpoint: str, port: int, timeout=60): """ Get an opensearch client from an endpoint and port @@ -796,7 +833,7 @@ def query_index(opensearch: OpenSearch, index_name: str, body: dict, def bulk_index(opensearch: OpenSearch, index_name: str, body: List): - return opensearch.bulk(index=index_name, body=body, timeout='5m') + return opensearch.bulk(index=index_name, body=body) def get_segment_stats(opensearch: OpenSearch, index_name: str): return opensearch.indices.segments(index=index_name) diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json index b8f591176..7e8ddda8e 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json @@ -3,7 +3,8 @@ "index": { "knn": true, "number_of_shards": 24, - "number_of_replicas": 1 + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 3445634b2..ba8850e1d 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,24 +1,30 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Relaxed Filter Test" test_id: "Faiss HNSW Relaxed Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/relaxed-filter/index.json + index_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query_with_filter k: 100 r: 1 @@ -26,9 +32,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json index b8f591176..7e8ddda8e 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json @@ -3,7 +3,8 @@ "index": { "knn": true, "number_of_shards": 24, - "number_of_replicas": 1 + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index bf02144ac..94f4073c7 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Restrictive Filter Test" test_id: "Faiss HNSW Restrictive Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/index.json + index_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -22,6 +23,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query_with_filter k: 100 r: 1 @@ -29,9 +32,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-spec.json + filter_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json index b8f591176..7e8ddda8e 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json @@ -3,7 +3,8 @@ "index": { "knn": true, "number_of_shards": 24, - "number_of_replicas": 1 + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml index f3e976cf3..c4740acf5 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml @@ -1,25 +1,28 @@ -endpoint: localhost +endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Test" test_id: "Faiss HNSW Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge index_name: target_index max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -27,6 +30,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml index dd88affc3..f573ede9c 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "index workflow" -num_runs: 10 +port: [PORT] +test_name: "Faiss HNSW PQ Test" +test_id: "Faiss HNSW PQ Test" +num_runs: 3 show_runs: false setup: - name: delete_index index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-hnswpq/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,19 +29,24 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-hnswpq/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-hnswpq/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -48,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json new file mode 100644 index 000000000..ade7fa377 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json new file mode 100644 index 000000000..51ae89877 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json @@ -0,0 +1,9 @@ +{ + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist": 128, + "nprobes": 8 + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json new file mode 100644 index 000000000..3e04d12c4 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json @@ -0,0 +1,42 @@ +{ + "bool": + { + "should": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 70 + } + } + }, + { + "term": + { + "color": "green" + } + }, + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "yellow" + } + }, + { + "term": + { + "taste": "sweet" + } + } + ] + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml new file mode 100644 index 000000000..adb25a04d --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml @@ -0,0 +1,64 @@ +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss IVF Relaxed Filter Test" +test_id: "Faiss IVF Relaxed Filter Test" +num_runs: 3 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 + neighbors_dataset: neighbors_filter_5 + filter_spec: release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json new file mode 100644 index 000000000..137fac9d8 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json new file mode 100644 index 000000000..ade7fa377 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json new file mode 100644 index 000000000..51ae89877 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json @@ -0,0 +1,9 @@ +{ + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist": 128, + "nprobes": 8 + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json new file mode 100644 index 000000000..9e6356f1c --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json @@ -0,0 +1,44 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 60 + } + } + }, + { + "term": + { + "taste": "bitter" + } + }, + { + "bool": + { + "should": + [ + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "green" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml new file mode 100644 index 000000000..bad047eab --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml @@ -0,0 +1,64 @@ +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss IVF restrictive Filter Test" +test_id: "Faiss IVF restrictive Filter Test" +num_runs: 3 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 + neighbors_dataset: neighbors_filter_4 + filter_spec: release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json new file mode 100644 index 000000000..804a5707e --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml index ce6b78867..367c42594 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml @@ -1,20 +1,21 @@ -endpoint: [END-POINT] -test_name: "index-workflow" -test_id: "index workflow" -num_runs: 10 +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss IVF" +test_id: "Faiss IVF" +num_runs: 3 show_runs: false setup: - name: delete_index index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-ivf/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/[PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,19 +29,24 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-ivf/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-ivf/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -48,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml index dd88affc3..c3f63348b 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "index workflow" -num_runs: 10 +port: [PORT] +test_name: "Faiss IVF PQ Test" +test_id: "Faiss IVF PQ Test" +num_runs: 3 show_runs: false setup: - name: delete_index index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-ivfpq/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,19 +29,24 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-ivfpq/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-ivfpq/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -48,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json index fecde0392..3e04d12c4 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json @@ -34,9 +34,9 @@ { "term": { - "color": "sweet" + "taste": "sweet" } } ] } -} \ No newline at end of file +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 62d032b00..3bbb99a0f 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,24 +1,28 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW Relaxed Filter Test" test_id: "Lucene HNSW Relaxed Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/index.json + index_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 - name: query_with_filter k: 100 r: 1 @@ -26,9 +30,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index d7f451a48..aa4c5193f 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW Restrictive Filter Test" test_id: "Lucene HNSW Restrictive Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/k-NN/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json + index_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -29,9 +30,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: /home/ec2-user/k-NN/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json + filter_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml index 96b991325..b253ee08e 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml @@ -1,22 +1,26 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "Index workflow" -num_runs: 10 +port: [PORT] +test_name: "Lucene HNSW" +test_id: "Lucene HNSW" +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/lucene-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 - name: query k: 100 r: 1 @@ -24,6 +28,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json index fa88192f6..eb714c5c8 100644 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json @@ -4,7 +4,7 @@ "knn": true, "number_of_shards": 24, "number_of_replicas": 1, - "knn.algo_param.ef_search": 256 + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml index 96b991325..94ad9b131 100644 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml @@ -1,22 +1,28 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "Index workflow" -num_runs: 10 +port: [PORT] +test_name: "Nmslib HNSW Test" +test_id: "Nmslib HNSW Test" +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/nmslib-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -24,6 +30,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/run_all_tests.sh b/benchmarks/perf-tool/release-configs/run_all_tests.sh new file mode 100755 index 000000000..5f684bd6f --- /dev/null +++ b/benchmarks/perf-tool/release-configs/run_all_tests.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -e + +# Description: +# Run a performance test for release +# Dataset should be available in perf-tool/dataset before running this script +# +# Example: +# ./run-test.sh --endpoint localhost +# +# Usage: +# ./run-test.sh \ +# --endpoint +# --port 80 \ +# --num-runs 3 \ +# --outputs ~/outputs + +while [ "$1" != "" ]; do + case $1 in + -url | --endpoint ) shift + ENDPOINT=$1 + ;; + -p | --port ) shift + PORT=$1 + ;; + -n | --num-runs ) shift + NUM_RUNS=$1 + ;; + -o | --outputs ) shift + OUTPUTS=$1 + ;; + * ) echo "Unknown parameter" + echo $1 + exit 1 + ;; + esac + shift +done + +if [ ! -n "$ENDPOINT" ]; then + echo "--endpoint should be specified" + exit +fi + +if [ ! -n "$PORT" ]; then + PORT=80 + echo "--port is not specified. Using default values $PORT" +fi + +if [ ! -n "$NUM_RUNS" ]; then + NUM_RUNS=3 + echo "--num-runs is not specified. Using default values $NUM_RUNS" +fi + +if [ ! -n "$OUTPUTS" ]; then + OUTPUTS="$HOME/outputs" + echo "--outputs is not specified. Using default values $OUTPUTS" +fi + + +curl -X PUT "http://$ENDPOINT:$PORT/_cluster/settings?pretty" -H 'Content-Type: application/json' -d' +{ + "persistent" : { + "knn.algo_param.index_thread_qty" : 4 + } +} +' + +TESTS="./release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/faiss-hnsw/test.yml +./release-configs/faiss-hnswpq/test.yml +./release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/faiss-ivf/test.yml +./release-configs/faiss-ivfpq/test.yml +./release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/lucene-hnsw/test.yml +./release-configs/nmslib-hnsw/test.yml" + +if [ ! -d $OUTPUTS ] +then + mkdir $OUTPUTS +fi + +for TEST in $TESTS +do + ORG_FILE=$TEST + NEW_FILE="$ORG_FILE.tmp" + OUT_FILE=$(grep test_id $ORG_FILE | cut -d':' -f2 | sed -r 's/^ "|"$//g' | sed 's/ /_/g') + echo "cp $ORG_FILE $NEW_FILE" + cp $ORG_FILE $NEW_FILE + sed -i "/^endpoint:/c\endpoint: $ENDPOINT" $NEW_FILE + sed -i "/^port:/c\port: $PORT" $NEW_FILE + sed -i "/^num_runs:/c\num_runs: $NUM_RUNS" $NEW_FILE + python3 knn-perf-tool.py test $NEW_FILE $OUTPUTS/$OUT_FILE + #Sleep for 1 min to cool down cpu from the previous run + sleep 60 +done diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml index c8fb42ec4..027ba8683 100644 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml +++ b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml @@ -43,6 +43,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 10 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 diff --git a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml index deea1ad47..6d96bf80c 100644 --- a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml +++ b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml @@ -21,6 +21,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 10 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1