-
Notifications
You must be signed in to change notification settings - Fork 129
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Increment version to 3.0.0-SNAPSHOT (#498)
* Increment version to 3.0.0-SNAPSHOT Signed-off-by: Naveen Tatikonda <[email protected]> * Nomenclature changes from Whitelist to Allowlist Signed-off-by: Naveen Tatikonda <[email protected]> * Update lucene92 package Signed-off-by: Naveen Tatikonda <[email protected]> * Add KNN940Codec based on Lucene94 codec Signed-off-by: Naveen Tatikonda <[email protected]> * Update Source URL for unreleased version 2.4.0 Signed-off-by: Naveen Tatikonda <[email protected]> * Address Review Comments Signed-off-by: Naveen Tatikonda <[email protected]> Signed-off-by: Naveen Tatikonda <[email protected]>
- Loading branch information
1 parent
2c33ad8
commit ce68025
Showing
15 changed files
with
245 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 62 additions & 0 deletions
62
src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.codec.KNN940Codec; | ||
|
|
||
| import lombok.Builder; | ||
| import org.apache.lucene.codecs.CompoundFormat; | ||
| import org.apache.lucene.codecs.Codec; | ||
| import org.apache.lucene.codecs.DocValuesFormat; | ||
| import org.apache.lucene.codecs.FilterCodec; | ||
| import org.apache.lucene.codecs.KnnVectorsFormat; | ||
| import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; | ||
| import org.opensearch.knn.index.codec.KNNFormatFacade; | ||
| import org.opensearch.knn.index.codec.KNNFormatFactory; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate; | ||
|
|
||
| public class KNN940Codec extends FilterCodec { | ||
| private static final String KNN940 = "KNN940Codec"; | ||
| private final KNNFormatFacade knnFormatFacade; | ||
| private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; | ||
|
|
||
| /** | ||
| * No arg constructor that uses Lucene94 as the delegate | ||
| */ | ||
| public KNN940Codec() { | ||
| this(createKNN94DefaultDelegate(), new KNN940PerFieldKnnVectorsFormat(Optional.empty())); | ||
| } | ||
|
|
||
| /** | ||
| * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec | ||
| * and a unique name to this ctor. | ||
| * | ||
| * @param delegate codec that will perform all operations this codec does not override | ||
| * @param knnVectorsFormat per field format for KnnVector | ||
| */ | ||
| @Builder | ||
| protected KNN940Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { | ||
| super(KNN940, delegate); | ||
| knnFormatFacade = KNNFormatFactory.createKNN940Format(delegate); | ||
| perFieldKnnVectorsFormat = knnVectorsFormat; | ||
| } | ||
|
|
||
| @Override | ||
| public DocValuesFormat docValuesFormat() { | ||
| return knnFormatFacade.docValuesFormat(); | ||
| } | ||
|
|
||
| @Override | ||
| public CompoundFormat compoundFormat() { | ||
| return knnFormatFacade.compoundFormat(); | ||
| } | ||
|
|
||
| @Override | ||
| public KnnVectorsFormat knnVectorsFormat() { | ||
| return perFieldKnnVectorsFormat; | ||
| } | ||
| } |
74 changes: 74 additions & 0 deletions
74
src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.codec.KNN940Codec; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.extern.log4j.Log4j2; | ||
| import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat; | ||
| import org.apache.lucene.codecs.KnnVectorsFormat; | ||
| import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; | ||
| import org.opensearch.index.mapper.MapperService; | ||
| import org.opensearch.knn.common.KNNConstants; | ||
| import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; | ||
|
|
||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * Class provides per field format implementation for Lucene Knn vector type | ||
| */ | ||
| @AllArgsConstructor | ||
| @Log4j2 | ||
| public class KNN940PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { | ||
|
|
||
| private final Optional<MapperService> mapperService; | ||
|
|
||
| @Override | ||
| public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { | ||
| if (isNotKnnVectorFieldType(field)) { | ||
| log.debug( | ||
| "Initialize KNN vector format for field [{}] with default params [max_connections] = \"{}\" and [beam_width] = \"{}\"", | ||
| field, | ||
| Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN, | ||
| Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH | ||
| ); | ||
| return new Lucene94HnswVectorsFormat(); | ||
| } | ||
| var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( | ||
| () -> new IllegalStateException( | ||
| String.format("Cannot read field type for field [%s] because mapper service is not available", field) | ||
| ) | ||
| ).fieldType(field); | ||
| var params = type.getKnnMethodContext().getMethodComponent().getParameters(); | ||
| int maxConnections = getMaxConnections(params); | ||
| int beamWidth = getBeamWidth(params); | ||
| log.debug( | ||
| "Initialize KNN vector format for field [{}] with params [max_connections] = \"{}\" and [beam_width] = \"{}\"", | ||
| field, | ||
| maxConnections, | ||
| beamWidth | ||
| ); | ||
| return new Lucene94HnswVectorsFormat(maxConnections, beamWidth); | ||
| } | ||
|
|
||
| private boolean isNotKnnVectorFieldType(final String field) { | ||
| return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); | ||
| } | ||
|
|
||
| private int getMaxConnections(final Map<String, Object> params) { | ||
| if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { | ||
| return (int) params.get(KNNConstants.METHOD_PARAMETER_M); | ||
| } | ||
| return Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN; | ||
| } | ||
|
|
||
| private int getBeamWidth(final Map<String, Object> params) { | ||
| if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { | ||
| return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); | ||
| } | ||
| return Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.codec.KNN940Codec; | ||
|
|
||
| import org.apache.lucene.codecs.Codec; | ||
| import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; | ||
| import org.opensearch.index.mapper.MapperService; | ||
| import org.opensearch.knn.index.codec.KNNCodecTestCase; | ||
| import java.io.IOException; | ||
| import java.util.Optional; | ||
| import java.util.concurrent.ExecutionException; | ||
| import java.util.function.Function; | ||
|
|
||
| import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate; | ||
|
|
||
| public class KNN940CodecTests extends KNNCodecTestCase { | ||
|
|
||
| public void testMultiFieldsKnnIndex() throws Exception { | ||
| testMultiFieldsKnnIndex(KNN940Codec.builder().delegate(createKNN94DefaultDelegate()).build()); | ||
| } | ||
|
|
||
| public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { | ||
| testBuildFromModelTemplate((KNN940Codec.builder().delegate(createKNN94DefaultDelegate()).build())); | ||
| } | ||
|
|
||
| public void testKnnVectorIndex() throws Exception { | ||
| Function<MapperService, PerFieldKnnVectorsFormat> perFieldKnnVectorsFormatProvider = ( | ||
| mapperService) -> new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService)); | ||
|
|
||
| Function<PerFieldKnnVectorsFormat, Codec> knnCodecProvider = (knnVectorFormat) -> KNN940Codec.builder() | ||
| .delegate(createKNN94DefaultDelegate()) | ||
| .knnVectorsFormat(knnVectorFormat) | ||
| .build(); | ||
|
|
||
| testKnnVectorIndex(knnCodecProvider, perFieldKnnVectorsFormatProvider); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.