-
Notifications
You must be signed in to change notification settings - Fork 127
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added support for Efficient Pre-filtering for Faiss Engine. The chang…
…es include (#936) * Enabled the efficient filtering support for Faiss Engine (#907) * Fixed the ef_search default value for faiss HNSW with filters and updated the perf-tool to include Faiss HNSW tests (#926) * Added exact search for cases when filteredIds < k to improve the recall for exact search (#928) * Improved Exact Search to return only K results and added client side latency metric for query Benchmarks (#933) * Added Integration Tests and Unit test for Efficient Filtering for Faiss Engine (#934) Signed-off-by: Navneet Verma <[email protected]>
- Loading branch information
Showing
45 changed files
with
1,146 additions
and
123 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| { | ||
| "settings": { | ||
| "index": { | ||
| "knn": true, | ||
| "number_of_shards": 24, | ||
| "number_of_replicas": 1 | ||
| } | ||
| }, | ||
| "mappings": { | ||
| "properties": { | ||
| "target_field": { | ||
| "type": "knn_vector", | ||
| "dimension": 128, | ||
| "method": { | ||
| "name": "hnsw", | ||
| "space_type": "l2", | ||
| "engine": "faiss", | ||
| "parameters": { | ||
| "ef_construction": 256, | ||
| "m": 16 | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
...ks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| { | ||
| "bool": | ||
| { | ||
| "should": | ||
| [ | ||
| { | ||
| "range": | ||
| { | ||
| "age": | ||
| { | ||
| "gte": 30, | ||
| "lte": 70 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "term": | ||
| { | ||
| "color": "green" | ||
| } | ||
| }, | ||
| { | ||
| "term": | ||
| { | ||
| "color": "blue" | ||
| } | ||
| }, | ||
| { | ||
| "term": | ||
| { | ||
| "color": "yellow" | ||
| } | ||
| }, | ||
| { | ||
| "term": | ||
| { | ||
| "color": "sweet" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| } |
34 changes: 34 additions & 0 deletions
34
...rks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| endpoint: [ENDPOINT] | ||
| test_name: "Faiss HNSW Relaxed Filter Test" | ||
| test_id: "Faiss HNSW Relaxed Filter Test" | ||
| num_runs: 10 | ||
| show_runs: false | ||
| steps: | ||
| - name: delete_index | ||
| index_name: target_index | ||
| - name: create_index | ||
| index_name: target_index | ||
| index_spec: [INDEX_SPEC_PATH]/relaxed-filter/index.json | ||
| - name: ingest_multi_field | ||
| index_name: target_index | ||
| field_name: target_field | ||
| bulk_size: 500 | ||
| dataset_format: hdf5 | ||
| dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 | ||
| attributes_dataset_name: attributes | ||
| attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] | ||
| - name: refresh_index | ||
| index_name: target_index | ||
| - name: query_with_filter | ||
| k: 100 | ||
| r: 1 | ||
| calculate_recall: true | ||
| index_name: target_index | ||
| field_name: target_field | ||
| dataset_format: hdf5 | ||
| dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 | ||
| neighbors_format: hdf5 | ||
| neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 | ||
| neighbors_dataset: neighbors_filter_5 | ||
| filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json | ||
| filter_type: FILTER |
26 changes: 26 additions & 0 deletions
26
benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| { | ||
| "settings": { | ||
| "index": { | ||
| "knn": true, | ||
| "number_of_shards": 24, | ||
| "number_of_replicas": 1 | ||
| } | ||
| }, | ||
| "mappings": { | ||
| "properties": { | ||
| "target_field": { | ||
| "type": "knn_vector", | ||
| "dimension": 128, | ||
| "method": { | ||
| "name": "hnsw", | ||
| "space_type": "l2", | ||
| "engine": "faiss", | ||
| "parameters": { | ||
| "ef_construction": 256, | ||
| "m": 16 | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
44 changes: 44 additions & 0 deletions
44
...tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| { | ||
| "bool": | ||
| { | ||
| "must": | ||
| [ | ||
| { | ||
| "range": | ||
| { | ||
| "age": | ||
| { | ||
| "gte": 30, | ||
| "lte": 60 | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "term": | ||
| { | ||
| "taste": "bitter" | ||
| } | ||
| }, | ||
| { | ||
| "bool": | ||
| { | ||
| "should": | ||
| [ | ||
| { | ||
| "term": | ||
| { | ||
| "color": "blue" | ||
| } | ||
| }, | ||
| { | ||
| "term": | ||
| { | ||
| "color": "green" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| } |
Oops, something went wrong.